diff --git a/docs/changelog/104037.yaml b/docs/changelog/104037.yaml new file mode 100644 index 0000000000000..b939353163d87 --- /dev/null +++ b/docs/changelog/104037.yaml @@ -0,0 +1,7 @@ +pr: 104037 +summary: "Add support for time, byte, and percent units in `NumberFieldMapper`" +area: "Mapping" +type: feature +issues: + - 65432 + - 31244 diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc index 26774c7091d27..fc44249e85119 100644 --- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc @@ -238,7 +238,7 @@ POST /sales/_search?size=0 "reason" : "[1:82] [date_histogram] failed to parse field [fixed_interval]", "caused_by" : { "type" : "illegal_argument_exception", - "reason" : "failed to parse setting [date_histogram.fixedInterval] with value [2w] as a time value: unit is missing or unrecognized", + "reason" : "failed to parse [date_histogram.fixedInterval] with value [2w] as a time value: unit is missing or unrecognized", "stack_trace" : "java.lang.IllegalArgumentException: failed to parse setting [date_histogram.fixedInterval] with value [2w] as a time value: unit is missing or unrecognized" } } @@ -581,7 +581,7 @@ For example, the offset of `+19d` will result in buckets with names like `2022-0 Increasing the offset to `+20d`, each document will appear in a bucket for the previous month, with all bucket keys ending with the same day of the month, as normal. -However, further increasing to `+28d`, +However, further increasing to `+28d`, what used to be a February bucket has now become `"2022-03-01"`. [source,console,id=datehistogram-aggregation-offset-example-28d] diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index 64cb499a9cd4e..8fd8e6debf3ae 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -378,13 +378,13 @@ Whenever durations need to be specified, e.g. for a `timeout` parameter, the dur the unit, like `2d` for 2 days. The supported units are: [horizontal] -`d`:: Days -`h`:: Hours -`m`:: Minutes -`s`:: Seconds -`ms`:: Milliseconds -`micros`:: Microseconds -`nanos`:: Nanoseconds +Days:: `d` +Hours:: `h` +Minutes:: `m` +Seconds:: `s` +Milliseconds:: `ms` +Microseconds:: `us` or `micros` +Nanoseconds:: `ns` or `nanos` [[size-units]] [discrete] diff --git a/docs/reference/mapping/params/meta.asciidoc b/docs/reference/mapping/params/meta.asciidoc index 7db73ba7a8529..4cc54c83b3cce 100644 --- a/docs/reference/mapping/params/meta.asciidoc +++ b/docs/reference/mapping/params/meta.asciidoc @@ -37,12 +37,33 @@ can follow these same metadata conventions to get a better out-of-the-box experience with your data. unit:: ++ +-- +The unit associated with a numeric field, preferably a case-sensitive http://unitsofmeasure.org/ucum.html[UCUM symbol]. +When a well-known unit is used, term and range queries support conversions within the same unit. +Also supports some non-UCUM symbols as well-known units for convenience and compatibility. +Well-known units: - The unit associated with a numeric field: `"percent"`, `"byte"` or a - <>. By default, a field does not have a unit. - Only valid for numeric fields. The convention for percents is to use - value `1` to mean `100%`. +|=== +|Unit |Value of `unit` |Supported unit symbols in queries +|Byte size +|`By` (preferred), `byte` +|<> + +|Time +|`ns`, `us`, `ms`, `s`, `min`, `h`, `d` +|<> + +|Percent (`0.01` means `1%`) +|`%` (preferred), `percent` +|`%` +|=== + +According to [UCUM](https://unitsofmeasure.org/ucum#para-50), symbols commonly used as units that are no real units of measurement can be used in curly brace expressions. +Examples: `{requests}, `{connections}`, `{threads}` + +-- metric_type:: The metric type of a numeric field: `"gauge"` or `"counter"`. A gauge is a diff --git a/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java b/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java index 2b178efe2cd0f..762e20adf086a 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java +++ b/libs/core/src/main/java/org/elasticsearch/core/TimeValue.java @@ -329,33 +329,37 @@ public String getStringRep() { }; } - public static TimeValue parseTimeValue(String sValue, String settingName) { - Objects.requireNonNull(settingName); + public static TimeValue parseTimeValue(String sValue, String fieldName) { + Objects.requireNonNull(fieldName); Objects.requireNonNull(sValue); - return parseTimeValue(sValue, null, settingName); + return parseTimeValue(sValue, null, fieldName); } - public static TimeValue parseTimeValue(String sValue, TimeValue defaultValue, String settingName) { - settingName = Objects.requireNonNull(settingName); + public static TimeValue parseTimeValue(String sValue, TimeValue defaultValue, String fieldName) { + fieldName = Objects.requireNonNull(fieldName); if (sValue == null) { return defaultValue; } final String normalized = sValue.toLowerCase(Locale.ROOT).trim(); if (normalized.endsWith("nanos")) { - return new TimeValue(parse(sValue, normalized, "nanos", settingName), TimeUnit.NANOSECONDS); + return new TimeValue(parse(sValue, normalized, "nanos", fieldName), TimeUnit.NANOSECONDS); + } else if (normalized.endsWith("ns")) { + return new TimeValue(parse(sValue, normalized, "ns", fieldName), TimeUnit.NANOSECONDS); } else if (normalized.endsWith("micros")) { - return new TimeValue(parse(sValue, normalized, "micros", settingName), TimeUnit.MICROSECONDS); + return new TimeValue(parse(sValue, normalized, "micros", fieldName), TimeUnit.MICROSECONDS); + } else if (normalized.endsWith("us")) { + return new TimeValue(parse(sValue, normalized, "us", fieldName), TimeUnit.MICROSECONDS); } else if (normalized.endsWith("ms")) { - return new TimeValue(parse(sValue, normalized, "ms", settingName), TimeUnit.MILLISECONDS); + return new TimeValue(parse(sValue, normalized, "ms", fieldName), TimeUnit.MILLISECONDS); } else if (normalized.endsWith("s")) { - return new TimeValue(parse(sValue, normalized, "s", settingName), TimeUnit.SECONDS); + return new TimeValue(parse(sValue, normalized, "s", fieldName), TimeUnit.SECONDS); } else if (sValue.endsWith("m")) { // parsing minutes should be case-sensitive as 'M' means "months", not "minutes"; this is the only special case. - return new TimeValue(parse(sValue, normalized, "m", settingName), TimeUnit.MINUTES); + return new TimeValue(parse(sValue, normalized, "m", fieldName), TimeUnit.MINUTES); } else if (normalized.endsWith("h")) { - return new TimeValue(parse(sValue, normalized, "h", settingName), TimeUnit.HOURS); + return new TimeValue(parse(sValue, normalized, "h", fieldName), TimeUnit.HOURS); } else if (normalized.endsWith("d")) { - return new TimeValue(parse(sValue, normalized, "d", settingName), TimeUnit.DAYS); + return new TimeValue(parse(sValue, normalized, "d", fieldName), TimeUnit.DAYS); } else if (normalized.matches("-0*1")) { return TimeValue.MINUS_ONE; } else if (normalized.matches("0+")) { @@ -363,20 +367,20 @@ public static TimeValue parseTimeValue(String sValue, TimeValue defaultValue, St } else { // Missing units: throw new IllegalArgumentException( - "failed to parse setting [" + settingName + "] with value [" + sValue + "] as a time value: unit is missing or unrecognized" + "failed to parse [" + fieldName + "] with value [" + sValue + "] as a time value: unit is missing or unrecognized" ); } } - private static long parse(final String initialInput, final String normalized, final String suffix, String settingName) { + private static long parse(final String initialInput, final String normalized, final String suffix, String fieldName) { final String s = normalized.substring(0, normalized.length() - suffix.length()).trim(); try { final long value = Long.parseLong(s); if (value < -1) { // -1 is magic, but reject any other negative values throw new IllegalArgumentException( - "failed to parse setting [" - + settingName + "failed to parse [" + + fieldName + "] with value [" + initialInput + "] as a time value: negative durations are not supported" diff --git a/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java index e3273c34bb32e..94be2b058e396 100644 --- a/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java +++ b/libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java @@ -132,7 +132,7 @@ public void testFractionalTimeValues() { } private String randomTimeUnit() { - return randomFrom("nanos", "micros", "ms", "s", "m", "h", "d"); + return randomFrom("nanos", "ns", "micros", "us", "ms", "s", "m", "h", "d"); } public void testFailOnUnknownUnits() { @@ -216,7 +216,7 @@ public void testRejectsNegativeValuesDuringParsing() { assertThat( ex.getMessage(), equalTo( - "failed to parse setting [" + "failed to parse [" + settingName + "] with value [" + negativeTimeValueString diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java index 8ca6e71b4a516..cdaa78de27420 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/BytesProcessorTests.java @@ -51,7 +51,7 @@ public void testTooLarge() { String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "8912pb"); Processor processor = newProcessor(fieldName, randomBoolean(), fieldName); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> processor.execute(ingestDocument)); - assertThat(exception.getMessage(), equalTo("failed to parse setting [Ingest Field] with value [8912pb] as a size in bytes")); + assertThat(exception.getMessage(), equalTo("failed to parse [Ingest Field] with value [8912pb] as a size in bytes")); assertThat(exception.getCause().getMessage(), containsString("Values greater than 9223372036854775807 bytes are not supported")); } @@ -60,7 +60,7 @@ public void testNotBytes() { String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, "junk"); Processor processor = newProcessor(fieldName, randomBoolean(), fieldName); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> processor.execute(ingestDocument)); - assertThat(exception.getMessage(), equalTo("failed to parse setting [Ingest Field] with value [junk]")); + assertThat(exception.getMessage(), equalTo("failed to parse [Ingest Field] with value [junk]")); } public void testMissingUnits() { @@ -78,7 +78,7 @@ public void testFractional() throws Exception { processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(fieldName, expectedResultType()), equalTo(1126L)); assertWarnings( - "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [1.1kb] found for setting " + "[Ingest Field]" + "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [1.1kb] found for " + "[Ingest Field]" ); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index e6c77b7b50c09..adecd9a646cfb 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.UnitOfMeasurement; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.fielddata.FieldData; @@ -180,6 +181,11 @@ private Parameter getMetric() { return metric; } + public Builder meta(Map meta) { + this.meta.setValue(meta); + return this; + } + @Override protected Parameter[] getParameters() { return new Parameter[] { indexed, hasDocValues, stored, ignoreMalformed, meta, scalingFactor, coerce, nullValue, metric }; @@ -210,6 +216,7 @@ public static final class ScaledFloatFieldType extends SimpleMappedFieldType { private final Double nullValue; private final TimeSeriesParams.MetricType metricType; private final IndexMode indexMode; + private final UnitOfMeasurement unit; public ScaledFloatFieldType( String name, @@ -227,6 +234,7 @@ public ScaledFloatFieldType( this.nullValue = nullValue; this.metricType = metricType; this.indexMode = indexMode; + this.unit = UnitOfMeasurement.of(meta.get("unit")); } public ScaledFloatFieldType(String name, double scalingFactor) { @@ -259,6 +267,7 @@ public boolean isSearchable() { @Override public Query termQuery(Object value, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); + value = unit.tryConvert(value, name()); long scaledValue = Math.round(scale(value)); return NumberFieldMapper.NumberType.LONG.termQuery(name(), scaledValue, isIndexed()); } @@ -266,6 +275,7 @@ public Query termQuery(Object value, SearchExecutionContext context) { @Override public Query termsQuery(Collection values, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); + values = values.stream().map(v -> unit.tryConvert(v, name())).toList(); if (isIndexed()) { List scaledValues = new ArrayList<>(values.size()); for (Object value : values) { @@ -289,6 +299,7 @@ public Query rangeQuery( failIfNotIndexedNorDocValuesFallback(context); Long lo = null; if (lowerTerm != null) { + lowerTerm = unit.tryConvert(lowerTerm, name()); double dValue = scale(lowerTerm); if (includeLower == false) { dValue = Math.nextUp(dValue); @@ -297,6 +308,7 @@ public Query rangeQuery( } Long hi = null; if (upperTerm != null) { + upperTerm = unit.tryConvert(upperTerm, name()); double dValue = scale(upperTerm); if (includeUpper == false) { dValue = Math.nextDown(dValue); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java index 222f0f05d548d..6dda4f9ca9d59 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java @@ -15,6 +15,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; @@ -34,6 +35,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; @@ -81,6 +84,28 @@ public void testTermsQuery() { ); } + public void testTermQueryTimeUnitConversion() { + ScaledFloatFieldMapper.ScaledFloatFieldType ft = fieldMapperWithUnit("us", 100).fieldType(); + assertEquals(LongPoint.newExactQuery("field", TimeUnit.MILLISECONDS.toMicros(42) * 100), ft.termQuery("42ms", MOCK_CONTEXT)); + } + + public void testTermsQueryTimeUnitConversion() { + ScaledFloatFieldMapper.ScaledFloatFieldType ft = fieldMapperWithUnit("us", 100).fieldType(); + assertEquals( + LongPoint.newSetQuery("field", 100, 200, 300, 400_000), + ft.termsQuery(Arrays.asList(1, "2", "3us", "4ms"), MOCK_CONTEXT) + ); + } + + public void testRangeQueryUnitConversion() { + ScaledFloatFieldMapper.ScaledFloatFieldType ft = fieldMapperWithUnit("us", 100).fieldType(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", 100, 300000), + SortedNumericDocValuesField.newSlowRangeQuery("field", 100, 300000) + ); + assertEquals(expected, ft.rangeQuery("1us", "3ms", true, true, null, null, null, MOCK_CONTEXT)); + } + public void testRangeQuery() throws IOException { // make sure the accuracy loss of scaled floats only occurs at index time // this test checks that searching scaled floats yields the same results as @@ -232,4 +257,10 @@ public void testFetchSourceValue() throws IOException { .fieldType(); assertEquals(List.of(2.71), fetchSourceValue(nullValueMapper, "")); } + + private static ScaledFloatFieldMapper fieldMapperWithUnit(String unit, int scalingFactor) { + return new ScaledFloatFieldMapper.Builder("field", false, true, null).scalingFactor(scalingFactor) + .meta(Map.of("unit", unit)) + .build(MapperBuilderContext.root(false, false)); + } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/510_units.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/510_units.yml new file mode 100644 index 0000000000000..1d3fcb9352ddc --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/510_units.yml @@ -0,0 +1,170 @@ +setup: + - skip: + version: " - 8.12.99" + reason: Support for units in queries has been added in 8.13 +--- +"Time units": + - do: + indices.create: + index: test_index + body: + settings: + number_of_shards: 1 + mappings: + properties: + duration_us: + type: long + meta: + unit: us + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "test_index", "_id": "1"}}' + - '{"duration_us": 1000}' + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: + query: + term: + duration_us: 1ms + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: + query: + range: + duration_us: + gte: 1us + lte: 1ms + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: + query: + range: + duration_us: + gte: 1us + lt: 1ms + - match: { hits.total: 0 } + +--- +"Byte units": + - do: + indices.create: + index: test_index + body: + settings: + number_of_shards: 1 + mappings: + properties: + request_size: + type: long + meta: + unit: By + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "test_index", "_id": "1"}}' + - '{"request_size": 1024}' + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: + query: + term: + request_size: 1kb + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: + query: + range: + request_size: + gte: 1b + lte: 1kb + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: + query: + range: + request_size: + gte: 1b + lt: 1kb + - match: { hits.total: 0 } + + +--- +"Percent units": + - do: + indices.create: + index: test_index + body: + settings: + number_of_shards: 1 + mappings: + properties: + cpu_utilization: + type: double + meta: + unit: "%" + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "test_index", "_id": "1"}}' + - '{"cpu_utilization": 0.42}' + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: + query: + term: + cpu_utilization: "42%" + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: + query: + range: + cpu_utilization: + gte: "1%" + lte: "42%" + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: + query: + range: + cpu_utilization: + gte: "1%" + lt: "42%" + - match: { hits.total: 0 } + diff --git a/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java b/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java index 819e14c176975..00314ead52602 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/versioning/SimpleVersioningIT.java @@ -214,7 +214,7 @@ public void testRequireUnitsOnUpdateSettings() throws Exception { // expected assertTrue( iae.getMessage() - .contains("failed to parse setting [index.gc_deletes] with value [42] as a time value: unit is missing or unrecognized") + .contains("failed to parse [index.gc_deletes] with value [42] as a time value: unit is missing or unrecognized") ); } } diff --git a/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java b/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java index 3da0f437258c4..6cddd56f0e5cc 100644 --- a/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java +++ b/server/src/main/java/org/elasticsearch/common/unit/ByteSizeValue.java @@ -254,7 +254,7 @@ public static ByteSizeValue parseBytesSizeValue(String sValue, ByteSizeValue def } else { // Missing units: throw new ElasticsearchParseException( - "failed to parse setting [{}] with value [{}] as a size in bytes: unit is missing or unrecognized", + "failed to parse [{}] with value [{}] as a size in bytes: unit is missing or unrecognized", settingName, sValue ); @@ -266,14 +266,9 @@ private static ByteSizeValue parseBytes(String lowerSValue, String settingName, try { return ByteSizeValue.ofBytes(Long.parseLong(s)); } catch (NumberFormatException e) { - throw new ElasticsearchParseException("failed to parse setting [{}] with value [{}]", e, settingName, initialInput); + throw new ElasticsearchParseException("failed to parse [{}] with value [{}]", e, settingName, initialInput); } catch (IllegalArgumentException e) { - throw new ElasticsearchParseException( - "failed to parse setting [{}] with value [{}] as a size in bytes", - e, - settingName, - initialInput - ); + throw new ElasticsearchParseException("failed to parse [{}] with value [{}] as a size in bytes", e, settingName, initialInput); } } @@ -294,22 +289,17 @@ private static ByteSizeValue parse( DeprecationLoggerHolder.deprecationLogger.warn( DeprecationCategory.PARSING, "fractional_byte_values", - "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [{}] found for setting [{}]", + "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [{}] found for [{}]", initialInput, settingName ); return ByteSizeValue.ofBytes((long) (doubleValue * unit.toBytes(1))); } catch (final NumberFormatException ignored) { - throw new ElasticsearchParseException("failed to parse setting [{}] with value [{}]", e, settingName, initialInput); + throw new ElasticsearchParseException("failed to parse [{}] with value [{}]", e, settingName, initialInput); } } } catch (IllegalArgumentException e) { - throw new ElasticsearchParseException( - "failed to parse setting [{}] with value [{}] as a size in bytes", - e, - settingName, - initialInput - ); + throw new ElasticsearchParseException("failed to parse [{}] with value [{}] as a size in bytes", e, settingName, initialInput); } } diff --git a/server/src/main/java/org/elasticsearch/common/unit/UnitOfMeasurement.java b/server/src/main/java/org/elasticsearch/common/unit/UnitOfMeasurement.java new file mode 100644 index 0000000000000..2265fe6e8948c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/unit/UnitOfMeasurement.java @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.unit; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.core.TimeValue; + +import java.util.concurrent.TimeUnit; + +public interface UnitOfMeasurement { + + /** + * Creates a unit of measurement instance given a unit identifier that preferably should be a + * case-sensitive UCUM symbol. + * + * @param unit the unit identifier, preferably a case-sensitive UCUM symbol + * @return either a well-known unit of measurement that can do conversions within it's scale or a generic unknown unit + */ + static UnitOfMeasurement of(String unit) { + if (unit == null) { + return Unknown.NONE; + } + // because UCUM identifiers are unique, we can identify both the scale and the unit + // thus, there's no need for a separate parameter such as scale=time + // also supports some non-UCUM identifiers (such as byte) for convenience and compatibility reasons + // when the risk of collisions with actual UCUM units is very low + return switch (unit) { + case "ns" -> new Time(TimeUnit.NANOSECONDS, unit); + case "us" -> new Time(TimeUnit.MICROSECONDS, unit); + case "ms" -> new Time(TimeUnit.MILLISECONDS, unit); + case "s" -> new Time(TimeUnit.SECONDS, unit); + case "min" -> new Time(TimeUnit.MINUTES, unit); + case "h" -> new Time(TimeUnit.HOURS, unit); + case "d" -> new Time(TimeUnit.DAYS, unit); + case "By", "byte" -> new Byte(unit); + case "%", "percent" -> new Percent(unit); + default -> new Unknown(unit); + }; + } + + /** + * If the provided value is a {@link String} or {@link BytesRef} with a unit suffix, the value will be converted to this unit. + * If the provided value has a different type or doesn't contain a unit suffix, the original value is returned. + * + * @param value the value to convert, which may be a {@link String}, a {@link BytesRef} containing a string, or a {@link Number} + * @param fieldName the name of the field to convert. Used in error messages. + * @return the converted or the original value, depending on whether the provided value contains a unit suffix + * @throws org.elasticsearch.ElasticsearchParseException if the provided value contains a unit suffix but the value can't be parsed + */ + Object tryConvert(Object value, String fieldName); + + String unit(); + + abstract class AbstractUnitOfMeasurement implements UnitOfMeasurement { + + private final String unit; + + protected AbstractUnitOfMeasurement(String unit) { + this.unit = unit; + } + + public Object tryConvert(Object value, String fieldName) { + if (value instanceof BytesRef bValue) { + value = bValue.utf8ToString(); + } + if (value instanceof String sValue) { + String trimmed = sValue.trim(); + if (hasUnitSuffix(trimmed)) { + try { + return doConvert(trimmed, fieldName); + } catch (IllegalArgumentException e) { + throw new ElasticsearchParseException(e.getMessage(), e); + } + } + } + return value; + } + + private boolean hasUnitSuffix(String sValue) { + return Character.isDigit(sValue.charAt(sValue.length() - 1)) == false; + } + + abstract Number doConvert(String value, String fieldName); + + @Override + public String unit() { + return unit; + } + } + + class Unknown implements UnitOfMeasurement { + + private static final Unknown NONE = new Unknown(null); + private final String unit; + + private Unknown(String unit) { + this.unit = unit; + } + + @Override + public Object tryConvert(Object value, String fieldName) { + return value; + } + + @Override + public String unit() { + return unit; + } + } + + class Time extends AbstractUnitOfMeasurement { + + private final TimeUnit timeUnit; + + Time(TimeUnit timeUnit, String unit) { + super(unit); + this.timeUnit = timeUnit; + } + + @Override + public Number doConvert(String value, String fieldName) { + TimeValue timeValue = TimeValue.parseTimeValue(value, fieldName); + return timeUnit.convert(timeValue.duration(), timeValue.timeUnit()); + } + } + + class Byte extends AbstractUnitOfMeasurement { + + Byte(String unit) { + super(unit); + } + + @Override + public Number doConvert(String value, String fieldName) { + return ByteSizeValue.parseBytesSizeValue(value, fieldName).getBytes(); + } + } + + class Percent extends AbstractUnitOfMeasurement { + + protected Percent(String unit) { + super(unit); + } + + @Override + Number doConvert(String value, String fieldName) { + return RatioValue.parseRatioValue(value).getAsRatio(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index d25832a28d318..0e429730acb34 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.UnitOfMeasurement; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -249,6 +250,11 @@ public Builder allowMultipleValues(boolean allowMultipleValues) { return this; } + public Builder meta(Map meta) { + this.meta.setValue(meta); + return this; + } + @Override protected Parameter[] getParameters() { return new Parameter[] { @@ -1533,6 +1539,7 @@ public static class NumberFieldType extends SimpleMappedFieldType { private final boolean isDimension; private final MetricType metricType; private final IndexMode indexMode; + private final UnitOfMeasurement unit; public NumberFieldType( String name, @@ -1556,6 +1563,7 @@ public NumberFieldType( this.isDimension = isDimension; this.metricType = metricType; this.indexMode = indexMode; + this.unit = UnitOfMeasurement.of(meta.get("unit")); } NumberFieldType(String name, Builder builder) { @@ -1619,12 +1627,13 @@ public boolean isSearchable() { @Override public Query termQuery(Object value, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); - return type.termQuery(name(), value, isIndexed()); + return type.termQuery(name(), unit.tryConvert(value, name()), isIndexed()); } @Override public Query termsQuery(Collection values, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); + values = values.stream().map(v -> unit.tryConvert(v, name())).toList(); if (isIndexed()) { return type.termsQuery(name(), values); } else { @@ -1641,6 +1650,8 @@ public Query rangeQuery( SearchExecutionContext context ) { failIfNotIndexedNorDocValuesFallback(context); + lowerTerm = unit.tryConvert(lowerTerm, name()); + upperTerm = unit.tryConvert(upperTerm, name()); return type.rangeQuery(name(), lowerTerm, upperTerm, includeLower, includeUpper, hasDocValues(), context, isIndexed()); } diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java index e5863988a76e5..e6bb2ac6c5af8 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java @@ -116,7 +116,7 @@ public void testByteSizeSettingValidation() { assertNotNull(e.getCause()); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); final IllegalArgumentException cause = (IllegalArgumentException) e.getCause(); - final String expected = "failed to parse setting [a.byte.size] with value [12] as a size in bytes: unit is missing or unrecognized"; + final String expected = "failed to parse [a.byte.size] with value [12] as a size in bytes: unit is missing or unrecognized"; assertThat(cause, hasToString(containsString(expected))); assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "12b").build(), Settings.EMPTY)); assertThat(value.get(), equalTo(ByteSizeValue.ofBytes(12))); @@ -157,8 +157,7 @@ public void testMemorySize() { assertNotNull(ex.getCause()); assertThat(ex.getCause(), instanceOf(IllegalArgumentException.class)); final IllegalArgumentException cause = (IllegalArgumentException) ex.getCause(); - final String expected = - "failed to parse setting [a.byte.size] with value [12] as a size in bytes: unit is missing or unrecognized"; + final String expected = "failed to parse [a.byte.size] with value [12] as a size in bytes: unit is missing or unrecognized"; assertThat(cause, hasToString(containsString(expected))); } diff --git a/server/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java b/server/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java index f965d3eb16837..34a7edbab332f 100644 --- a/server/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java +++ b/server/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java @@ -122,12 +122,12 @@ public void testParsing() { public void testFailOnMissingUnits() { Exception e = expectThrows(ElasticsearchParseException.class, () -> ByteSizeValue.parseBytesSizeValue("23", "test")); - assertThat(e.getMessage(), containsString("failed to parse setting [test]")); + assertThat(e.getMessage(), containsString("failed to parse [test]")); } public void testFailOnUnknownUnits() { Exception e = expectThrows(ElasticsearchParseException.class, () -> ByteSizeValue.parseBytesSizeValue("23jw", "test")); - assertThat(e.getMessage(), containsString("failed to parse setting [test]")); + assertThat(e.getMessage(), containsString("failed to parse [test]")); } public void testFailOnEmptyParsing() { @@ -135,7 +135,7 @@ public void testFailOnEmptyParsing() { ElasticsearchParseException.class, () -> assertThat(ByteSizeValue.parseBytesSizeValue("", "emptyParsing").toString(), is("23kb")) ); - assertThat(e.getMessage(), containsString("failed to parse setting [emptyParsing]")); + assertThat(e.getMessage(), containsString("failed to parse [emptyParsing]")); } public void testFailOnEmptyNumberParsing() { @@ -143,12 +143,12 @@ public void testFailOnEmptyNumberParsing() { ElasticsearchParseException.class, () -> assertThat(ByteSizeValue.parseBytesSizeValue("g", "emptyNumberParsing").toString(), is("23b")) ); - assertThat(e.getMessage(), containsString("failed to parse setting [emptyNumberParsing] with value [g]")); + assertThat(e.getMessage(), containsString("failed to parse [emptyNumberParsing] with value [g]")); } public void testNoDotsAllowed() { Exception e = expectThrows(ElasticsearchParseException.class, () -> ByteSizeValue.parseBytesSizeValue("42b.", null, "test")); - assertThat(e.getMessage(), containsString("failed to parse setting [test]")); + assertThat(e.getMessage(), containsString("failed to parse [test]")); } public void testCompareEquality() { @@ -277,7 +277,7 @@ public void testParseInvalidValue() { ElasticsearchParseException.class, () -> ByteSizeValue.parseBytesSizeValue("-6" + unitSuffix, "test_setting") ); - assertEquals("failed to parse setting [test_setting] with value [-6" + unitSuffix + "] as a size in bytes", exception.getMessage()); + assertEquals("failed to parse [test_setting] with value [-6" + unitSuffix + "] as a size in bytes", exception.getMessage()); assertNotNull(exception.getCause()); assertEquals(IllegalArgumentException.class, exception.getCause().getClass()); } @@ -303,7 +303,7 @@ public void testParseInvalidNumber() throws IOException { () -> ByteSizeValue.parseBytesSizeValue("notANumber", "test") ); assertEquals( - "failed to parse setting [test] with value [notANumber] as a size in bytes: unit is missing or unrecognized", + "failed to parse [test] with value [notANumber] as a size in bytes: unit is missing or unrecognized", exception.getMessage() ); @@ -312,7 +312,7 @@ public void testParseInvalidNumber() throws IOException { ElasticsearchParseException.class, () -> ByteSizeValue.parseBytesSizeValue("notANumber" + unitSuffix, "test") ); - assertEquals("failed to parse setting [test] with value [notANumber" + unitSuffix + "]", exception.getMessage()); + assertEquals("failed to parse [test] with value [notANumber" + unitSuffix + "]", exception.getMessage()); } public void testParseFractionalNumber() throws IOException { @@ -321,9 +321,7 @@ public void testParseFractionalNumber() throws IOException { ByteSizeValue instance = ByteSizeValue.parseBytesSizeValue(fractionalValue, "test"); assertEquals(fractionalValue, instance.toString()); assertWarnings( - "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [" - + fractionalValue - + "] found for setting [test]" + "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [" + fractionalValue + "] found for [test]" ); } diff --git a/server/src/test/java/org/elasticsearch/common/unit/UnitOfMeasurementTests.java b/server/src/test/java/org/elasticsearch/common/unit/UnitOfMeasurementTests.java new file mode 100644 index 0000000000000..97297f433fdb1 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/unit/UnitOfMeasurementTests.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.unit; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.test.ESTestCase; + +public class UnitOfMeasurementTests extends ESTestCase { + + public void testNullUnit() { + UnitOfMeasurement unit = UnitOfMeasurement.of(null); + assertEquals("42thrd", unit.tryConvert("42thrd", "jvm_thread_count")); + } + + public void testUnknownUnit() { + UnitOfMeasurement unit = UnitOfMeasurement.of("{threads}"); + assertEquals("42thrd", unit.tryConvert("42thrd", "jvm_thread_count")); + } + + public void testByteUnit() { + UnitOfMeasurement unit = UnitOfMeasurement.of("By"); + assertEquals(42, unit.tryConvert(42, "request_bytes")); + assertEquals("42", unit.tryConvert("42", "request_bytes")); + assertEquals(42L, unit.tryConvert("42b ", "request_bytes")); + assertEquals(42L * 1024, unit.tryConvert("42 KB", "request_bytes")); + } + + public void testFractionalBytes() { + assertEquals((long) (1.1 * 1024), UnitOfMeasurement.of("By").tryConvert("1.1kb", "request_bytes")); + assertWarnings( + "Fractional bytes values are deprecated. Use non-fractional bytes values instead: [1.1kb] found for [request_bytes]" + ); + } + + public void testByteUnitInvalid() { + UnitOfMeasurement unit = UnitOfMeasurement.of("byte"); + assertEquals( + "failed to parse [request_bytes] with value [-42b] as a size in bytes", + expectThrows(ElasticsearchParseException.class, () -> unit.tryConvert("-42b", "request_bytes")).getMessage() + ); + assertEquals( + "failed to parse [request_bytes] with value [1by] as a size in bytes: unit is missing or unrecognized", + expectThrows(ElasticsearchParseException.class, () -> unit.tryConvert("1by", "request_bytes")).getMessage() + ); + assertEquals( + "failed to parse [request_bytes] with value [1foo] as a size in bytes: unit is missing or unrecognized", + expectThrows(ElasticsearchParseException.class, () -> unit.tryConvert("1foo", "request_bytes")).getMessage() + ); + } + + public void testTimeUnit() { + UnitOfMeasurement unit = UnitOfMeasurement.of("ns"); + assertEquals(42, unit.tryConvert(42, "duration")); + assertEquals("42", unit.tryConvert("42", "duration")); + assertEquals(42L, unit.tryConvert("42 ns", "duration")); + assertEquals(42L * 1000, unit.tryConvert("42US ", "duration")); + } + + public void testTimeUnitInvalid() { + UnitOfMeasurement unit = UnitOfMeasurement.of("s"); + assertEquals( + "failed to parse [duration] with value [-42s] as a time value: negative durations are not supported", + expectThrows(ElasticsearchParseException.class, () -> unit.tryConvert("-42s", "duration")).getMessage() + ); + assertEquals( + "failed to parse [duration] with value [1sec] as a time value: unit is missing or unrecognized", + expectThrows(ElasticsearchParseException.class, () -> unit.tryConvert("1sec", "duration")).getMessage() + ); + assertEquals( + "failed to parse [duration] with value [1foo] as a time value: unit is missing or unrecognized", + expectThrows(ElasticsearchParseException.class, () -> unit.tryConvert("1foo", "duration")).getMessage() + ); + assertEquals( + "failed to parse [1.1s], fractional time values are not supported", + expectThrows(ElasticsearchParseException.class, () -> unit.tryConvert("1.1s", "duration")).getMessage() + ); + } + + public void testPercent() { + UnitOfMeasurement unit = UnitOfMeasurement.of("%"); + assertEquals(42, unit.tryConvert(42, "ratio")); + assertEquals("42", unit.tryConvert("42", "ratio")); + assertEquals(0.42, unit.tryConvert("42%", "ratio")); + assertEquals(0.421, (double) unit.tryConvert("42.1%", "ratio"), 0.000001); + } + + public void testPercentInvalid() { + UnitOfMeasurement unit = UnitOfMeasurement.of("percent"); + assertEquals( + "Percentage should be in [0-100], got [200]", + expectThrows(ElasticsearchParseException.class, () -> unit.tryConvert("200%", "effort")).getMessage() + ); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index 3d95152c88cc5..6040681be38cd 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -716,7 +716,7 @@ public void testArchiveBrokenIndexSettings() { (e, ex) -> { assertThat(e.getKey(), equalTo("index.refresh_interval")); assertThat(e.getValue(), equalTo("-200")); - assertThat(ex, hasToString(containsString("failed to parse setting [index.refresh_interval] with value [-200]"))); + assertThat(ex, hasToString(containsString("failed to parse [index.refresh_interval] with value [-200]"))); } ); assertEquals("-200", settings.get("archived.index.refresh_interval")); diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java index fb83e817c052e..1602e50f687b4 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java @@ -461,7 +461,7 @@ private void assertTimeValueException(final IllegalArgumentException e, final St assertNotNull(e.getCause()); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); final IllegalArgumentException cause = (IllegalArgumentException) e.getCause(); - final String causeExpected = "failed to parse setting [" + final String causeExpected = "failed to parse [" + key + "] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized"; assertThat(cause, hasToString(containsString(causeExpected))); diff --git a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java index 2fa3216ad5556..310224548acb7 100644 --- a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java @@ -501,7 +501,7 @@ private void assertTimeValueException(final IllegalArgumentException e, final St assertNotNull(e.getCause()); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); final IllegalArgumentException cause = (IllegalArgumentException) e.getCause(); - final String causeExpected = "failed to parse setting [" + final String causeExpected = "failed to parse [" + key + "] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized"; assertThat(cause, hasToString(containsString(causeExpected))); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 40d1f2488749a..c8f591653055f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -32,6 +32,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.IOUtils; @@ -58,6 +59,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import static org.hamcrest.Matchers.containsString; @@ -162,6 +165,44 @@ public void testTermQuery() { ); } + public void testTermQueryTimeUnitConversion() { + NumberFieldType ft = numberFieldMapperWithUnit(NumberType.LONG, "us").fieldType(); + assertEquals(LongPoint.newExactQuery("field", TimeUnit.MILLISECONDS.toMicros(42)), ft.termQuery("42ms", MOCK_CONTEXT)); + } + + public void testTermsQueryTimeUnitConversion() { + NumberFieldType ft = numberFieldMapperWithUnit(NumberType.LONG, "us").fieldType(); + assertEquals(LongPoint.newSetQuery("field", 1, 2, 3, 4000), ft.termsQuery(Arrays.asList(1, "2", "3us", "4ms"), MOCK_CONTEXT)); + } + + public void testTermQueryTimeUnitNoUnits() { + NumberFieldType ft = numberFieldMapperWithUnit(NumberType.LONG, "us").fieldType(); + assertEquals(LongPoint.newExactQuery("field", 42), ft.termQuery(42, MOCK_CONTEXT)); + assertEquals(LongPoint.newExactQuery("field", 42), ft.termQuery(42.0, MOCK_CONTEXT)); + assertEquals(LongPoint.newExactQuery("field", 42), ft.termQuery("42", MOCK_CONTEXT)); + } + + public void testTermQueryTimeUnitInvalid() { + NumberFieldType ft = numberFieldMapperWithUnit(NumberType.LONG, "us").fieldType(); + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> ft.termQuery("42mb", MOCK_CONTEXT)); + assertEquals("failed to parse [field] with value [42mb] as a time value: unit is missing or unrecognized", e.getMessage()); + } + + public void testRangeQueryUnitConversion() { + NumberFieldType ft = numberFieldMapperWithUnit(NumberType.LONG, "us").fieldType(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", 1, 3000), + SortedNumericDocValuesField.newSlowRangeQuery("field", 1, 3000) + ); + assertEquals(expected, ft.rangeQuery("1us", "3ms", true, true, null, null, null, MOCK_CONTEXT)); + } + + private static NumberFieldMapper numberFieldMapperWithUnit(NumberType numberType, String unit) { + return new NumberFieldMapper.Builder("field", numberType, ScriptCompiler.NONE, false, true, IndexVersion.current(), null).meta( + Map.of("unit", unit) + ).build(MapperBuilderContext.root(false, false)); + } + public void testRangeQueryWithNegativeBounds() { MappedFieldType ftInt = new NumberFieldMapper.NumberFieldType("field", NumberType.INTEGER, randomBoolean()); assertEquals( diff --git a/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java index 7535f900ff2d1..97da71d74a436 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java @@ -244,7 +244,7 @@ public void testNegativeMinIndexBufferSize() { IllegalArgumentException.class, () -> new MockController(Settings.builder().put("indices.memory.min_index_buffer_size", "-6mb").build()) ); - assertEquals("failed to parse setting [indices.memory.min_index_buffer_size] with value [-6mb] as a size in bytes", e.getMessage()); + assertEquals("failed to parse [indices.memory.min_index_buffer_size] with value [-6mb] as a size in bytes", e.getMessage()); } @@ -254,8 +254,7 @@ public void testNegativeInterval() { () -> new MockController(Settings.builder().put("indices.memory.interval", "-42s").build()) ); assertEquals( - "failed to parse setting [indices.memory.interval] with value " - + "[-42s] as a time value: negative durations are not supported", + "failed to parse [indices.memory.interval] with value " + "[-42s] as a time value: negative durations are not supported", e.getMessage() ); @@ -267,7 +266,7 @@ public void testNegativeShardInactiveTime() { () -> new MockController(Settings.builder().put("indices.memory.shard_inactive_time", "-42s").build()) ); assertEquals( - "failed to parse setting [indices.memory.shard_inactive_time] with value " + "failed to parse [indices.memory.shard_inactive_time] with value " + "[-42s] as a time value: negative durations are not supported", e.getMessage() ); @@ -279,7 +278,7 @@ public void testNegativeMaxIndexBufferSize() { IllegalArgumentException.class, () -> new MockController(Settings.builder().put("indices.memory.max_index_buffer_size", "-6mb").build()) ); - assertEquals("failed to parse setting [indices.memory.max_index_buffer_size] with value [-6mb] as a size in bytes", e.getMessage()); + assertEquals("failed to parse [indices.memory.max_index_buffer_size] with value [-6mb] as a size in bytes", e.getMessage()); } diff --git a/server/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java b/server/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java index 22f6035e2b325..bd656e0754cb7 100644 --- a/server/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java @@ -68,7 +68,7 @@ public void testNegativeSetting() throws InterruptedException { public void testNegativeOneSetting() throws InterruptedException { String collector = randomAlphaOfLength(5); - final String timeValue = "-1" + randomFrom("", "d", "h", "m", "s", "ms", "nanos"); + final String timeValue = "-1" + randomFrom("", "d", "h", "m", "s", "ms", "us", "micros", "ns", "nanos"); Settings settings = Settings.builder().put("monitor.jvm.gc.collector." + collector + ".warn", timeValue).build(); execute(settings, (command, interval, name) -> null, e -> { assertThat(e, instanceOf(IllegalArgumentException.class)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java index 972ec1f47a904..5edb965bad8b8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java @@ -702,10 +702,7 @@ public void testFixedWithCalendar() throws IOException { ); assertThat( e.getMessage(), - equalTo( - "failed to parse setting [date_histogram.fixedInterval] with value [1w] as a time value: " - + "unit is missing or unrecognized" - ) + equalTo("failed to parse [date_histogram.fixedInterval] with value [1w] as a time value: " + "unit is missing or unrecognized") ); } @@ -1107,8 +1104,7 @@ public void testIllegalInterval() throws IOException { assertThat( e.getMessage(), equalTo( - "failed to parse setting [date_histogram.fixedInterval] with value [foobar] as a time value:" - + " unit is missing or unrecognized" + "failed to parse [date_histogram.fixedInterval] with value [foobar] as a time value:" + " unit is missing or unrecognized" ) ); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index e8fdf7e0205da..4a76719794c53 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -19,6 +19,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.UnitOfMeasurement; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -180,6 +181,11 @@ private Parameter getMetric() { return metric; } + public Builder meta(Map meta) { + this.meta.setValue(meta); + return this; + } + @Override protected Parameter[] getParameters() { return new Parameter[] { indexed, hasDocValues, stored, ignoreMalformed, nullValue, meta, dimension, metric }; @@ -218,6 +224,7 @@ public static final class UnsignedLongFieldType extends SimpleMappedFieldType { private final boolean isDimension; private final MetricType metricType; private final IndexMode indexMode; + private final UnitOfMeasurement unit; public UnsignedLongFieldType( String name, @@ -235,6 +242,7 @@ public UnsignedLongFieldType( this.isDimension = isDimension; this.metricType = metricType; this.indexMode = indexMode; + this.unit = UnitOfMeasurement.of(meta.get("unit")); } public UnsignedLongFieldType(String name) { @@ -254,6 +262,7 @@ public boolean mayExistInIndex(SearchExecutionContext context) { @Override public Query termQuery(Object value, SearchExecutionContext context) { failIfNotIndexed(); + value = unit.tryConvert(value, name()); Long longValue = parseTerm(value); if (longValue == null) { return new MatchNoDocsQuery(); @@ -264,6 +273,7 @@ public Query termQuery(Object value, SearchExecutionContext context) { @Override public Query termsQuery(Collection values, SearchExecutionContext context) { failIfNotIndexed(); + values = values.stream().map(v -> unit.tryConvert(v, name())).toList(); long[] lvalues = new long[values.size()]; int upTo = 0; for (Object value : values) { @@ -293,11 +303,13 @@ public Query rangeQuery( long l = Long.MIN_VALUE; long u = Long.MAX_VALUE; if (lowerTerm != null) { + lowerTerm = unit.tryConvert(lowerTerm, name()); Long lt = parseLowerRangeTerm(lowerTerm, includeLower); if (lt == null) return new MatchNoDocsQuery(); l = unsignedToSortableSignedLong(lt); } if (upperTerm != null) { + upperTerm = unit.tryConvert(upperTerm, name()); Long ut = parseUpperRangeTerm(upperTerm, includeUpper); if (ut == null) return new MatchNoDocsQuery(); u = unsignedToSortableSignedLong(ut); diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java index e5f85f8b87b12..762047e231598 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldTypeTests.java @@ -8,15 +8,21 @@ package org.elasticsearch.xpack.unsignedlong; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.BIGINTEGER_2_64_MINUS_ONE; import static org.elasticsearch.xpack.unsignedlong.UnsignedLongFieldMapper.UnsignedLongFieldType.parseLowerRangeTerm; @@ -97,6 +103,31 @@ public void testRangeQuery() { expectThrows(NumberFormatException.class, () -> ft.rangeQuery("18incorrectnumber", "18incorrectnumber", true, true, null)); } + public void testTermQueryTimeUnitConversion() { + UnsignedLongFieldType ft = fieldMapperWithUnit("us").fieldType(); + assertEquals( + LongPoint.newExactQuery("field", toUnsignedLong(TimeUnit.MILLISECONDS.toMicros(42))), + ft.termQuery("42ms", MOCK_CONTEXT) + ); + } + + public void testTermsQueryTimeUnitConversion() { + UnsignedLongFieldType ft = fieldMapperWithUnit("us").fieldType(); + assertEquals( + LongPoint.newSetQuery("field", toUnsignedLong(1), toUnsignedLong(2), toUnsignedLong(3), toUnsignedLong(4000)), + ft.termsQuery(Arrays.asList(1, "2", "3us", "4ms"), MOCK_CONTEXT) + ); + } + + public void testRangeQueryUnitConversion() { + UnsignedLongFieldType ft = fieldMapperWithUnit("us").fieldType(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", toUnsignedLong(1), toUnsignedLong(3000)), + SortedNumericDocValuesField.newSlowRangeQuery("field", toUnsignedLong(1), toUnsignedLong(3000)) + ); + assertEquals(expected, ft.rangeQuery("1us", "3ms", true, true, null, null, null, MOCK_CONTEXT)); + } + public void testParseTermForTermQuery() { // values that represent proper unsigned long number assertEquals(0L, parseTerm("0").longValue()); @@ -180,4 +211,14 @@ public void testFetchSourceValue() throws IOException { .fieldType(); assertEquals(List.of(BIGINTEGER_2_64_MINUS_ONE), fetchSourceValue(nullValueMapper, "")); } + + private static UnsignedLongFieldMapper fieldMapperWithUnit(String unit) { + return new UnsignedLongFieldMapper.Builder("field", false, null).meta(Map.of("unit", unit)) + .build(MapperBuilderContext.root(false, false)); + } + + private long toUnsignedLong(long l) { + assert l >= 0; + return Long.MIN_VALUE + l; + } } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkActionTests.java index 3d1034cd815e7..6ac5dac75cafb 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkActionTests.java @@ -75,7 +75,7 @@ public void testWrongInterval() { final RestRequest restRequest = createRestRequest(randomSystem().getSystem(), TEMPLATE_VERSION, "null"); final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> prepareRequest(restRequest)); - assertThat(exception.getMessage(), containsString("failed to parse setting [interval] with value [null]")); + assertThat(exception.getMessage(), containsString("failed to parse [interval] with value [null]")); } public void testMissingContent() { diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java index 555248fdfa3e2..a9783b6388174 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java @@ -658,8 +658,7 @@ public void testValidateFixedInterval() { assertThat( e.getMessage(), equalTo( - "failed to parse setting [date_histo.config.interval] with value " - + "[minute] as a time value: unit is missing or unrecognized" + "failed to parse [date_histo.config.interval] with value " + "[minute] as a time value: unit is missing or unrecognized" ) ); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSettingsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSettingsTests.java index 894bfc6e13d5d..51397ca297c33 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSettingsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSettingsTests.java @@ -331,7 +331,7 @@ public void testTimeSettingsWithDefault() { assertThat( exception.getMessage(), equalTo( - "failed to parse setting [" + "failed to parse [" + settingKey + "] with value [" + rejectedValue diff --git a/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java index 110a1fd24d0d3..b744130759a7d 100644 --- a/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java +++ b/x-pack/qa/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -309,6 +309,10 @@ protected static boolean runtimeifyMappingProperties(Map propert if (RUNTIME_TYPES.contains(type) == false) { continue; } + if (propertyMap.get("meta") instanceof Map meta && meta.containsKey("unit")) { + // units have an effect on query behavior that's currently not supported by runtime fields + continue; + } Map runtimeConfig = new HashMap<>(propertyMap); runtimeConfig.put("type", type); runtimeConfig.remove("store");