Add support for time, byte, and percent units in NumberFieldMapper

docs/changelog/104037.yaml

@@ -0,0 +1,7 @@
+pr: 104037
+summary: "Add support for time, byte, and percent units in `NumberFieldMapper`"
+area: "Mapping"
+type: feature
+issues:
+ - 65432
+ - 31244

docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc

@@ -238,7 +238,7 @@ POST /sales/_search?size=0
     "reason" : "[1:82] [date_histogram] failed to parse field [fixed_interval]",
     "caused_by" : {
       "type" : "illegal_argument_exception",
-      "reason" : "failed to parse setting [date_histogram.fixedInterval] with value [2w] as a time value: unit is missing or unrecognized",
+      "reason" : "failed to parse [date_histogram.fixedInterval] with value [2w] as a time value: unit is missing or unrecognized",
       "stack_trace" : "java.lang.IllegalArgumentException: failed to parse setting [date_histogram.fixedInterval] with value [2w] as a time value: unit is missing or unrecognized"
     }
   }
@@ -581,7 +581,7 @@ For example, the offset of `+19d` will result in buckets with names like `2022-0
 
 Increasing the offset to `+20d`, each document will appear in a bucket for the previous month,
 with all bucket keys ending with the same day of the month, as normal.
-However, further increasing to `+28d`, 
+However, further increasing to `+28d`,
 what used to be a February bucket has now become `"2022-03-01"`.
 
 [source,console,id=datehistogram-aggregation-offset-example-28d]

docs/reference/api-conventions.asciidoc

@@ -378,13 +378,13 @@ Whenever durations need to be specified, e.g. for a `timeout` parameter, the dur
 the unit, like `2d` for 2 days. The supported units are:
 
 [horizontal]
-`d`::       Days
-`h`::       Hours
-`m`::       Minutes
-`s`::       Seconds
-`ms`::      Milliseconds
-`micros`::  Microseconds
-`nanos`::   Nanoseconds
+Days::         `d`
+Hours::        `h`
+Minutes::      `m`
+Seconds::      `s`
+Milliseconds:: `ms`
+Microseconds:: `us` or `micros`
+Nanoseconds::  `ns` or `nanos`
 
 [[size-units]]
 [discrete]

docs/reference/mapping/params/meta.asciidoc

@@ -37,12 +37,33 @@ can follow these same metadata conventions to get a better out-of-the-box
 experience with your data.
 
 unit::
++
+--
+The unit associated with a numeric field, preferably a case-sensitive http://unitsofmeasure.org/ucum.html[UCUM symbol].
+When a well-known unit is used, term and range queries support conversions within the same unit.
+Also supports some non-UCUM symbols as well-known units for convenience and compatibility.
+Well-known units:
 
-  The unit associated with a numeric field: `"percent"`, `"byte"` or a
-  <<time-units,time unit>>. By default, a field does not have a unit.
-  Only valid for numeric fields. The convention for percents is to use
-  value `1` to mean `100%`.
+|===
+|Unit |Value of `unit` |Supported unit symbols in queries
 
+|Byte size
+|`By` (preferred), `byte`
+|<<byte-units, byte units>>
+
+|Time
+|`ns`, `us`, `ms`, `s`, `min`, `h`, `d`
+|<<time-units, time units>>
+
+|Percent (`0.01` means `1%`)
+|`%` (preferred), `percent`
+|`%`
+|===
+
+According to [UCUM](https://unitsofmeasure.org/ucum#para-50), symbols commonly used as units that are no real units of measurement can be used in curly brace expressions.
+Examples: `{requests}, `{connections}`, `{threads}`
+
+--
 metric_type::
 
   The metric type of a numeric field: `"gauge"` or `"counter"`. A gauge is a

libs/core/src/main/java/org/elasticsearch/core/TimeValue.java

@@ -329,54 +329,58 @@ public String getStringRep() {
         };
     }
 
-    public static TimeValue parseTimeValue(String sValue, String settingName) {
-        Objects.requireNonNull(settingName);
+    public static TimeValue parseTimeValue(String sValue, String fieldName) {
+        Objects.requireNonNull(fieldName);
         Objects.requireNonNull(sValue);
-        return parseTimeValue(sValue, null, settingName);
+        return parseTimeValue(sValue, null, fieldName);
     }
 
-    public static TimeValue parseTimeValue(String sValue, TimeValue defaultValue, String settingName) {
-        settingName = Objects.requireNonNull(settingName);
+    public static TimeValue parseTimeValue(String sValue, TimeValue defaultValue, String fieldName) {
+        fieldName = Objects.requireNonNull(fieldName);
         if (sValue == null) {
             return defaultValue;
         }
         final String normalized = sValue.toLowerCase(Locale.ROOT).trim();
         if (normalized.endsWith("nanos")) {
-            return new TimeValue(parse(sValue, normalized, "nanos", settingName), TimeUnit.NANOSECONDS);
+            return new TimeValue(parse(sValue, normalized, "nanos", fieldName), TimeUnit.NANOSECONDS);
+        } else if (normalized.endsWith("ns")) {
+            return new TimeValue(parse(sValue, normalized, "ns", fieldName), TimeUnit.NANOSECONDS);
         } else if (normalized.endsWith("micros")) {
-            return new TimeValue(parse(sValue, normalized, "micros", settingName), TimeUnit.MICROSECONDS);
+            return new TimeValue(parse(sValue, normalized, "micros", fieldName), TimeUnit.MICROSECONDS);
+        } else if (normalized.endsWith("us")) {
+            return new TimeValue(parse(sValue, normalized, "us", fieldName), TimeUnit.MICROSECONDS);
         } else if (normalized.endsWith("ms")) {
-            return new TimeValue(parse(sValue, normalized, "ms", settingName), TimeUnit.MILLISECONDS);
+            return new TimeValue(parse(sValue, normalized, "ms", fieldName), TimeUnit.MILLISECONDS);
         } else if (normalized.endsWith("s")) {
-            return new TimeValue(parse(sValue, normalized, "s", settingName), TimeUnit.SECONDS);
+            return new TimeValue(parse(sValue, normalized, "s", fieldName), TimeUnit.SECONDS);
         } else if (sValue.endsWith("m")) {
             // parsing minutes should be case-sensitive as 'M' means "months", not "minutes"; this is the only special case.
-            return new TimeValue(parse(sValue, normalized, "m", settingName), TimeUnit.MINUTES);
+            return new TimeValue(parse(sValue, normalized, "m", fieldName), TimeUnit.MINUTES);
         } else if (normalized.endsWith("h")) {
-            return new TimeValue(parse(sValue, normalized, "h", settingName), TimeUnit.HOURS);
+            return new TimeValue(parse(sValue, normalized, "h", fieldName), TimeUnit.HOURS);
         } else if (normalized.endsWith("d")) {
-            return new TimeValue(parse(sValue, normalized, "d", settingName), TimeUnit.DAYS);
+            return new TimeValue(parse(sValue, normalized, "d", fieldName), TimeUnit.DAYS);
         } else if (normalized.matches("-0*1")) {
             return TimeValue.MINUS_ONE;
         } else if (normalized.matches("0+")) {
             return TimeValue.ZERO;
         } else {
             // Missing units:
             throw new IllegalArgumentException(
-                "failed to parse setting [" + settingName + "] with value [" + sValue + "] as a time value: unit is missing or unrecognized"
+                "failed to parse [" + fieldName + "] with value [" + sValue + "] as a time value: unit is missing or unrecognized"
             );
         }
     }
 
-    private static long parse(final String initialInput, final String normalized, final String suffix, String settingName) {
+    private static long parse(final String initialInput, final String normalized, final String suffix, String fieldName) {
         final String s = normalized.substring(0, normalized.length() - suffix.length()).trim();
         try {
             final long value = Long.parseLong(s);
             if (value < -1) {
                 // -1 is magic, but reject any other negative values
                 throw new IllegalArgumentException(
-                    "failed to parse setting ["
-                        + settingName
+                    "failed to parse ["
+                        + fieldName
                         + "] with value ["
                         + initialInput
                         + "] as a time value: negative durations are not supported"

libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java

@@ -132,7 +132,7 @@ public void testFractionalTimeValues() {
     }
 
     private String randomTimeUnit() {
-        return randomFrom("nanos", "micros", "ms", "s", "m", "h", "d");
+        return randomFrom("nanos", "ns", "micros", "us", "ms", "s", "m", "h", "d");
     }
 
     public void testFailOnUnknownUnits() {
@@ -216,7 +216,7 @@ public void testRejectsNegativeValuesDuringParsing() {
         assertThat(
             ex.getMessage(),
             equalTo(
-                "failed to parse setting ["
+                "failed to parse ["
                     + settingName
                     + "] with value ["
                     + negativeTimeValueString

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java

@@ -51,7 +51,7 @@ public void testTooLarge() {
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "8912pb");
         Processor processor = newProcessor(fieldName, randomBoolean(), fieldName);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> processor.execute(ingestDocument));
-        assertThat(exception.getMessage(), equalTo("failed to parse setting [Ingest Field] with value [8912pb] as a size in bytes"));
+        assertThat(exception.getMessage(), equalTo("failed to parse [Ingest Field] with value [8912pb] as a size in bytes"));
         assertThat(exception.getCause().getMessage(), containsString("Values greater than 9223372036854775807 bytes are not supported"));
     }
 
@@ -60,7 +60,7 @@ public void testNotBytes() {
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "junk");
         Processor processor = newProcessor(fieldName, randomBoolean(), fieldName);
         ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> processor.execute(ingestDocument));
-        assertThat(exception.getMessage(), equalTo("failed to parse setting [Ingest Field] with value [junk]"));
+        assertThat(exception.getMessage(), equalTo("failed to parse [Ingest Field] with value [junk]"));
     }
 
     public void testMissingUnits() {
@@ -78,7 +78,7 @@ public void testFractional() throws Exception {
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(fieldName, expectedResultType()), equalTo(1126L));
         assertWarnings(
-            "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [1.1kb] found for setting " + "[Ingest Field]"
+            "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [1.1kb] found for " + "[Ingest Field]"
         );
     }
 }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.UnitOfMeasurement;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.fielddata.FieldData;
@@ -180,6 +181,11 @@ private Parameter<TimeSeriesParams.MetricType> getMetric() {
             return metric;
         }
 
+        public Builder meta(Map<String, String> meta) {
+            this.meta.setValue(meta);
+            return this;
+        }
+
         @Override
         protected Parameter<?>[] getParameters() {
             return new Parameter<?>[] { indexed, hasDocValues, stored, ignoreMalformed, meta, scalingFactor, coerce, nullValue, metric };
@@ -210,6 +216,7 @@ public static final class ScaledFloatFieldType extends SimpleMappedFieldType {
         private final Double nullValue;
         private final TimeSeriesParams.MetricType metricType;
         private final IndexMode indexMode;
+        private final UnitOfMeasurement unit;
 
         public ScaledFloatFieldType(
             String name,
@@ -227,6 +234,7 @@ public ScaledFloatFieldType(
             this.nullValue = nullValue;
             this.metricType = metricType;
             this.indexMode = indexMode;
+            this.unit = UnitOfMeasurement.of(meta.get("unit"));
         }
 
         public ScaledFloatFieldType(String name, double scalingFactor) {
@@ -259,13 +267,15 @@ public boolean isSearchable() {
         @Override
         public Query termQuery(Object value, SearchExecutionContext context) {
             failIfNotIndexedNorDocValuesFallback(context);
+            value = unit.tryConvert(value, name());
             long scaledValue = Math.round(scale(value));
             return NumberFieldMapper.NumberType.LONG.termQuery(name(), scaledValue, isIndexed());
         }
 
         @Override
         public Query termsQuery(Collection<?> values, SearchExecutionContext context) {
             failIfNotIndexedNorDocValuesFallback(context);
+            values = values.stream().map(v -> unit.tryConvert(v, name())).toList();
             if (isIndexed()) {
                 List<Long> scaledValues = new ArrayList<>(values.size());
                 for (Object value : values) {
@@ -289,6 +299,7 @@ public Query rangeQuery(
             failIfNotIndexedNorDocValuesFallback(context);
             Long lo = null;
             if (lowerTerm != null) {
+                lowerTerm = unit.tryConvert(lowerTerm, name());
                 double dValue = scale(lowerTerm);
                 if (includeLower == false) {
                     dValue = Math.nextUp(dValue);
@@ -297,6 +308,7 @@ public Query rangeQuery(
             }
             Long hi = null;
             if (upperTerm != null) {
+                upperTerm = unit.tryConvert(upperTerm, name());
                 double dValue = scale(upperTerm);
                 if (includeUpper == false) {
                     dValue = Math.nextDown(dValue);

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java

@@ -15,6 +15,7 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
@@ -34,6 +35,8 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.containsString;
 
@@ -81,6 +84,28 @@ public void testTermsQuery() {
         );
     }
 
+    public void testTermQueryTimeUnitConversion() {
+        ScaledFloatFieldMapper.ScaledFloatFieldType ft = fieldMapperWithUnit("us", 100).fieldType();
+        assertEquals(LongPoint.newExactQuery("field", TimeUnit.MILLISECONDS.toMicros(42) * 100), ft.termQuery("42ms", MOCK_CONTEXT));
+    }
+
+    public void testTermsQueryTimeUnitConversion() {
+        ScaledFloatFieldMapper.ScaledFloatFieldType ft = fieldMapperWithUnit("us", 100).fieldType();
+        assertEquals(
+            LongPoint.newSetQuery("field", 100, 200, 300, 400_000),
+            ft.termsQuery(Arrays.asList(1, "2", "3us", "4ms"), MOCK_CONTEXT)
+        );
+    }
+
+    public void testRangeQueryUnitConversion() {
+        ScaledFloatFieldMapper.ScaledFloatFieldType ft = fieldMapperWithUnit("us", 100).fieldType();
+        Query expected = new IndexOrDocValuesQuery(
+            LongPoint.newRangeQuery("field", 100, 300000),
+            SortedNumericDocValuesField.newSlowRangeQuery("field", 100, 300000)
+        );
+        assertEquals(expected, ft.rangeQuery("1us", "3ms", true, true, null, null, null, MOCK_CONTEXT));
+    }
+
     public void testRangeQuery() throws IOException {
         // make sure the accuracy loss of scaled floats only occurs at index time
         // this test checks that searching scaled floats yields the same results as
@@ -232,4 +257,10 @@ public void testFetchSourceValue() throws IOException {
             .fieldType();
         assertEquals(List.of(2.71), fetchSourceValue(nullValueMapper, ""));
     }
+
+    private static ScaledFloatFieldMapper fieldMapperWithUnit(String unit, int scalingFactor) {
+        return new ScaledFloatFieldMapper.Builder("field", false, true, null).scalingFactor(scalingFactor)
+            .meta(Map.of("unit", unit))
+            .build(MapperBuilderContext.root(false, false));
+    }
 }