From 43a676fccc70c907b9e3ea67e0232b904cec36b3 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 27 Jul 2023 16:51:38 +0200 Subject: [PATCH 001/125] fix: hash the tsid if required A Lucene limitation on doc values for UTF-8 fields does not allow us to write string fields whose size is larger then 32K. This limits our ability to map more than a certain number of dimension fields for time series indices. To overcome this limitation we hash the tsid if the size of the fields exceeds the limit and prepend the hash with a prefix that makes it easier to query fields and investigate how many documents are using an hahed tsid. --- .../index/mapper/TimeSeriesIdFieldMapper.java | 43 +++++++------------ .../mapper/TsidExtractingIdFieldMapper.java | 9 ++++ .../index/mapper/KeywordFieldMapperTests.java | 7 ++- .../mapper/TimeSeriesIdFieldMapperTests.java | 25 ++++++----- 4 files changed, 43 insertions(+), 41 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index a85ba6d0e9a45..7fc05e23bbc96 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -35,7 +35,6 @@ import java.time.ZoneId; import java.util.Collections; import java.util.LinkedHashMap; -import java.util.Locale; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; @@ -56,11 +55,6 @@ public class TimeSeriesIdFieldMapper extends MetadataFieldMapper { * Lucene's writer for utf-8 doc values. */ private static final int LIMIT = ByteBlockPool.BYTE_BLOCK_SIZE - 2; - /** - * Maximum length of the name of dimension. We picked this so that we could - * comfortable fit 16 dimensions inside {@link #LIMIT}. - */ - private static final int DIMENSION_NAME_LIMIT = 512; /** * The maximum length of any single dimension. We picked this so that we could * comfortable fit 16 dimensions inside {@link #LIMIT}. This should be quite @@ -68,6 +62,7 @@ public class TimeSeriesIdFieldMapper extends MetadataFieldMapper { * hundred bytes each, but we're being paranoid here. */ private static final int DIMENSION_VALUE_LIMIT = 1024; + public static final int TSID_HASH_SENTINEL = 0xBAADCAFE; @Override public FieldMapper.Builder getMergeBuilder() { @@ -165,10 +160,13 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { */ public static Map decodeTsid(StreamInput in) { try { - int size = in.readVInt(); - Map result = new LinkedHashMap(size); + int sizeOrTsidHashSentinel = in.readVInt(); + if (sizeOrTsidHashSentinel == TSID_HASH_SENTINEL) { + return Collections.singletonMap(TimeSeriesIdFieldMapper.NAME, in.readBytesRef().utf8ToString()); + } + Map result = new LinkedHashMap<>(sizeOrTsidHashSentinel); - for (int i = 0; i < size; i++) { + for (int i = 0; i < sizeOrTsidHashSentinel; i++) { String name = in.readBytesRef().utf8ToString(); int type = in.read(); @@ -215,24 +213,18 @@ public BytesReference build() throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(dimensions.size()); for (Map.Entry entry : dimensions.entrySet()) { - BytesRef fieldName = entry.getKey(); - if (fieldName.length > DIMENSION_NAME_LIMIT) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "Dimension name must be less than [%d] bytes but [%s] was [%s].", - DIMENSION_NAME_LIMIT, - fieldName.utf8ToString(), - fieldName.length - ) - ); - } + final BytesRef fieldName = entry.getKey(); out.writeBytesRef(fieldName); entry.getValue().writeTo(out); } - BytesReference timeSeriesId = out.bytes(); + final BytesReference timeSeriesId = out.bytes(); if (timeSeriesId.length() > LIMIT) { - throw new IllegalArgumentException(NAME + " longer than [" + LIMIT + "] bytes [" + timeSeriesId.length() + "]."); + // NOTE: we hash the _tsid only if necessary, which is if the field does not fit Lucene UTF-8 doc values + try (BytesStreamOutput hashOut = new BytesStreamOutput()) { + hashOut.writeVInt(TSID_HASH_SENTINEL); + hashOut.writeBytesRef(TsidExtractingIdFieldMapper.hashTsid(timeSeriesId.toBytesRef())); + return hashOut.bytes(); + } } return timeSeriesId; } @@ -247,11 +239,6 @@ public void addString(String fieldName, BytesRef utf8Value) { * so it's easier for folks to reason about the space taken up. Mostly * it'll be smaller too. */ - if (utf8Value.length > DIMENSION_VALUE_LIMIT) { - throw new IllegalArgumentException( - "Dimension fields must be less than [" + DIMENSION_VALUE_LIMIT + "] bytes but was [" + utf8Value.length + "]." - ); - } out.writeBytesRef(utf8Value); add(fieldName, out.bytes()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index e8e4b6909fc4e..0428cdca65b45 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -42,6 +42,9 @@ public class TsidExtractingIdFieldMapper extends IdFieldMapper { public static final TsidExtractingIdFieldMapper INSTANCE = new TsidExtractingIdFieldMapper(); public static final TypeParser PARSER = new FixedTypeParser(MappingParserContext::idFieldMapper); + // NOTE: we use a prefix when hashing the tsid field so to be able later on (for instance for debugging purposes) + // to query documents whose tsid has been hashed. Using a prefix allows us to query using the prefix. + public static final String TSID_HASH_PREFIX = "hash-"; static final class IdFieldType extends TermBasedFieldType { IdFieldType() { @@ -169,6 +172,12 @@ public static String createId( return id; } + public static BytesRef hashTsid(final BytesRef tsid) { + final Hash128 hash = new Hash128(); + MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash); + return new BytesRef(TSID_HASH_PREFIX + hash); + } + @Override public String documentDescription(DocumentParserContext context) { /* diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 94e2506d2b2a7..ba963701d57f4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -15,6 +15,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.tests.analysis.MockLowerCaseFilter; @@ -396,9 +397,11 @@ public void testDimensionExtraLongKeyword() throws IOException { Exception e = expectThrows( DocumentParsingException.class, - () -> mapper.parse(source(b -> b.field("field", randomAlphaOfLengthBetween(1025, 2048)))) + () -> mapper.parse( + source(b -> b.field("field", randomAlphaOfLengthBetween(IndexWriter.MAX_TERM_LENGTH - 100, IndexWriter.MAX_TERM_LENGTH))) + ) ); - assertThat(e.getCause().getMessage(), containsString("Dimension fields must be less than [1024] bytes but was")); + assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] is missing")); } public void testConfigureSimilarity() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index f7bac4f303e5a..7dc49fd96320b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -158,11 +158,11 @@ public void testKeywordTooLong() throws IOException { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); })); - Exception e = expectThrows( - DocumentParsingException.class, - () -> parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52)).field("@timestamp", "2021-10-01")) + ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); + assertMap( + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + matchesMap().entry("a", "more_than_1024_bytes".repeat(52)) ); - assertThat(e.getCause().getMessage(), equalTo("Dimension fields must be less than [1024] bytes but was [1040].")); } public void testKeywordTooLongUtf8() throws IOException { @@ -171,11 +171,11 @@ public void testKeywordTooLongUtf8() throws IOException { })); String theWordLong = "長い"; - Exception e = expectThrows( - DocumentParsingException.class, - () -> parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200)).field("@timestamp", "2021-10-01")) + ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); + assertMap( + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + matchesMap().entry("a", theWordLong.repeat(200)) ); - assertThat(e.getCause().getMessage(), equalTo("Dimension fields must be less than [1024] bytes but was [1200].")); } public void testKeywordNull() throws IOException { @@ -471,13 +471,16 @@ public void testVeryLarge() throws IOException { })); String large = "many words ".repeat(80); - Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> { + ParsedDocument doc = parseDocument(docMapper, b -> { b.field("b", "foo"); for (int i = 0; i < 100; i++) { b.field("d" + i, large); } - })); - assertThat(e.getCause().getMessage(), equalTo("_tsid longer than [32766] bytes [88698].")); + }); + assertMap( + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + matchesMap().entry("_tsid", "hash-0x3777619b3ef94123dd58ef302f9d439d") + ); } /** From 1d900e41170b4d37f60cd792c4753477cb647e5e Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 28 Jul 2023 12:19:33 +0200 Subject: [PATCH 002/125] fix: comment document id assertion --- .../support/TimeSeriesDimensionsLimitIT.java | 51 ++++++++----------- .../mapper/TsidExtractingIdFieldMapper.java | 6 +-- 2 files changed, 23 insertions(+), 34 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java index 379539c3130c5..0eb9b7788cd83 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java @@ -14,7 +14,6 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESIntegTestCase; @@ -44,27 +43,19 @@ public void testDimensionFieldNameLimit() throws IOException { () -> List.of("routing_field"), dimensionFieldLimit ); - final Exception ex = expectThrows( - DocumentParsingException.class, - () -> client().prepareIndex("test") - .setSource( - "routing_field", - randomAlphaOfLength(10), - dimensionFieldName, - randomAlphaOfLength(1024), - "gauge", - randomIntBetween(10, 20), - "@timestamp", - Instant.now().toEpochMilli() - ) - .get() - ); - assertThat( - ex.getCause().getMessage(), - equalTo( - "Dimension name must be less than [512] bytes but [" + dimensionFieldName + "] was [" + dimensionFieldName.length() + "]." + final IndexResponse response = client().prepareIndex("test") + .setSource( + "routing_field", + randomAlphaOfLength(10), + dimensionFieldName, + randomAlphaOfLength(1024), + "gauge", + randomIntBetween(10, 20), + "@timestamp", + Instant.now().toEpochMilli() ) - ); + .get(); + assertEquals(RestStatus.CREATED.getStatus(), response.status().getStatus()); } public void testDimensionFieldValueLimit() throws IOException { @@ -76,16 +67,14 @@ public void testDimensionFieldValueLimit() throws IOException { dimensionFieldLimit ); long startTime = Instant.now().toEpochMilli(); - client().prepareIndex("test") + final IndexResponse response1 = client().prepareIndex("test") .setSource("field", randomAlphaOfLength(1024), "gauge", randomIntBetween(10, 20), "@timestamp", startTime) .get(); - final Exception ex = expectThrows( - DocumentParsingException.class, - () -> client().prepareIndex("test") - .setSource("field", randomAlphaOfLength(1025), "gauge", randomIntBetween(10, 20), "@timestamp", startTime + 1) - .get() - ); - assertThat(ex.getCause().getMessage(), equalTo("Dimension fields must be less than [1024] bytes but was [1025].")); + final IndexResponse response2 = client().prepareIndex("test") + .setSource("field", randomAlphaOfLength(1025), "gauge", randomIntBetween(10, 20), "@timestamp", startTime + 1) + .get(); + assertEquals(RestStatus.CREATED.getStatus(), response1.status().getStatus()); + assertEquals(RestStatus.CREATED.getStatus(), response2.status().getStatus()); } public void testTotalNumberOfDimensionFieldsLimit() { @@ -167,8 +156,8 @@ public void testTotalDimensionFieldsSizeLuceneLimitPlusOne() throws IOException for (int i = 0; i < dimensionFieldLimit; i++) { source.put(dimensionFieldNames.get(i), randomAlphaOfLength(1024)); } - final Exception ex = expectThrows(DocumentParsingException.class, () -> client().prepareIndex("test").setSource(source).get()); - assertEquals("_tsid longer than [32766] bytes [33903].", ex.getCause().getMessage()); + final IndexResponse response = client().prepareIndex("test").setSource(source).get(); + assertEquals(RestStatus.CREATED.getStatus(), response.status().getStatus()); } private void createTimeSeriesIndex( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 0428cdca65b45..5203ae362d6d3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -122,9 +122,9 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra * it always must pass. */ IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); - assert context.getDynamicMappers().isEmpty() == false - || context.getDynamicRuntimeFields().isEmpty() == false - || id.equals(indexRouting.createId(TimeSeriesIdFieldMapper.decodeTsid(tsid), suffix)); + // assert context.getDynamicMappers().isEmpty() == false + // || context.getDynamicRuntimeFields().isEmpty() == false + // || id.equals(indexRouting.createId(TimeSeriesIdFieldMapper.decodeTsid(tsid), suffix)); assert context.getDynamicMappers().isEmpty() == false || context.getDynamicRuntimeFields().isEmpty() == false || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); From 0c24a1228362fa7a6242a6fbb8806c2b587dea5a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 2 Aug 2023 18:22:56 +0200 Subject: [PATCH 003/125] fix: base 64 encode the hash --- .../TimeSeriesAggregationsUnlimitedDimensionsIT.java | 2 ++ .../index/mapper/TimeSeriesIdFieldMapper.java | 10 +++++++--- .../index/mapper/TsidExtractingIdFieldMapper.java | 6 +++--- .../index/mapper/TimeSeriesIdFieldMapperTests.java | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java new file mode 100644 index 0000000000000..d2e9e2e97d02f --- /dev/null +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -0,0 +1,2 @@ +package org.elasticsearch.aggregations.bucket;public class TimeSeriesAggregationsUnlimitedDimensionsIT { +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 7fc05e23bbc96..723f12f6496e5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.net.InetAddress; import java.time.ZoneId; +import java.util.Base64; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; @@ -61,8 +62,9 @@ public class TimeSeriesIdFieldMapper extends MetadataFieldMapper { * comfortable given that dimensions are typically going to be less than a * hundred bytes each, but we're being paranoid here. */ - private static final int DIMENSION_VALUE_LIMIT = 1024; public static final int TSID_HASH_SENTINEL = 0xBAADCAFE; + private static final Base64.Encoder TSID_BASE64_ENCODER = Base64.getUrlEncoder().withoutPadding(); + public static final String TSID_HASH_PREFIX = "hash-"; @Override public FieldMapper.Builder getMergeBuilder() { @@ -162,7 +164,7 @@ public static Map decodeTsid(StreamInput in) { try { int sizeOrTsidHashSentinel = in.readVInt(); if (sizeOrTsidHashSentinel == TSID_HASH_SENTINEL) { - return Collections.singletonMap(TimeSeriesIdFieldMapper.NAME, in.readBytesRef().utf8ToString()); + return Collections.singletonMap(TimeSeriesIdFieldMapper.NAME, in.readBytesRef()); } Map result = new LinkedHashMap<>(sizeOrTsidHashSentinel); @@ -221,8 +223,10 @@ public BytesReference build() throws IOException { if (timeSeriesId.length() > LIMIT) { // NOTE: we hash the _tsid only if necessary, which is if the field does not fit Lucene UTF-8 doc values try (BytesStreamOutput hashOut = new BytesStreamOutput()) { + final byte[] buffer = new byte[16]; hashOut.writeVInt(TSID_HASH_SENTINEL); - hashOut.writeBytesRef(TsidExtractingIdFieldMapper.hashTsid(timeSeriesId.toBytesRef())); + TsidExtractingIdFieldMapper.hashTsid(timeSeriesId.toBytesRef(), buffer); + hashOut.writeBytesRef(new BytesRef(TSID_HASH_PREFIX + TSID_BASE64_ENCODER.encodeToString(buffer))); return hashOut.bytes(); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 5203ae362d6d3..3ede5b0256750 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -44,7 +44,6 @@ public class TsidExtractingIdFieldMapper extends IdFieldMapper { public static final TypeParser PARSER = new FixedTypeParser(MappingParserContext::idFieldMapper); // NOTE: we use a prefix when hashing the tsid field so to be able later on (for instance for debugging purposes) // to query documents whose tsid has been hashed. Using a prefix allows us to query using the prefix. - public static final String TSID_HASH_PREFIX = "hash-"; static final class IdFieldType extends TermBasedFieldType { IdFieldType() { @@ -172,10 +171,11 @@ public static String createId( return id; } - public static BytesRef hashTsid(final BytesRef tsid) { + public static void hashTsid(final BytesRef tsid, byte[] buffer) { final Hash128 hash = new Hash128(); MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash); - return new BytesRef(TSID_HASH_PREFIX + hash); + ByteUtils.writeLongLE(hash.h1, buffer, 0); + ByteUtils.writeLongLE(hash.h2, buffer, 8); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 7dc49fd96320b..14dcf6118231c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -479,7 +479,7 @@ public void testVeryLarge() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "hash-0x3777619b3ef94123dd58ef302f9d439d") + matchesMap().entry("_tsid", new BytesRef("hash-I0H5PpthdzedQ50vMO9Y3Q")) ); } From 672e27c4704b0fba13b45d8404a58654fc9c3121 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 2 Aug 2023 18:23:26 +0200 Subject: [PATCH 004/125] test: a few aggregations (cardinality not working) --- ...riesAggregationsUnlimitedDimensionsIT.java | 223 +++++++++++++++++- 1 file changed, 222 insertions(+), 1 deletion(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java index d2e9e2e97d02f..aa0dee493beeb 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -1,2 +1,223 @@ -package org.elasticsearch.aggregations.bucket;public class TimeSeriesAggregationsUnlimitedDimensionsIT { +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.aggregations.bucket; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.aggregations.AggregationIntegTestCase; +import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries; +import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.InternalCardinality; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Supplier; + +@ESIntegTestCase.ClusterScope(numDataNodes = 3, numClientNodes = 1, supportsDedicatedMasters = true) +@ESIntegTestCase.SuiteScopeTestCase +public class TimeSeriesAggregationsUnlimitedDimensionsIT extends AggregationIntegTestCase { + private static int numberOfDimensions; + private static int numberOfDocs; + + @Override + public void setupSuiteScopeCluster() throws Exception { + numberOfDimensions = randomIntBetween(25, 99); + final XContentBuilder mapping = timeSeriesIndexMapping(); + long startMillis = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); + long endMillis = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-02T00:00:00Z"); + numberOfDocs = randomIntBetween(1000, 2000); + final Iterator timestamps = getTimestamps(startMillis, endMillis, numberOfDocs); + // NOTE: use also the last (changing) dimension so to make sure documents are not indexed all in the same shard. + final String[] routingDimensions = new String[] { "dim_1", "dim_" + (numberOfDimensions - 1) }; + assertTrue(prepareTimeSeriesIndex(mapping, startMillis, endMillis, routingDimensions).isAcknowledged()); + + logger.info("Dimensions: " + numberOfDimensions + " docs: " + numberOfDocs + " start: " + startMillis + " end: " + endMillis); + + // NOTE: we need the tsid to be larger than 32 Kb + final String fooDimValue = "foo_" + randomAlphaOfLengthBetween(2048, 2048 + 128); + final String barDimValue = "bar_" + randomAlphaOfLengthBetween(2048, 2048 + 256); + final String bazDimValue = "baz_" + randomAlphaOfLengthBetween(2048, 2048 + 512); + + final BulkRequestBuilder bulkIndexRequest = client().prepareBulk(); + for (int docId = 0; docId < numberOfDocs; docId++) { + final XContentBuilder document = timeSeriesDocument(fooDimValue, barDimValue, bazDimValue, docId, timestamps::next); + bulkIndexRequest.add(client().prepareIndex("index").setOpType(DocWriteRequest.OpType.CREATE).setSource(document)); + } + BulkResponse bulkIndexResponse = bulkIndexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + assertFalse(bulkIndexResponse.hasFailures()); + assertEquals(RestStatus.OK.getStatus(), client().admin().indices().prepareFlush("index").get().getStatus().getStatus()); + } + + private static XContentBuilder timeSeriesDocument( + final String fooDimValue, + final String barDimValue, + final String bazDimValue, + int docId, + final Supplier timestampSupplier + ) throws IOException { + final XContentBuilder docSource = XContentFactory.jsonBuilder(); + docSource.startObject(); + // NOTE: we assign dimensions in such a way that almost all of them have the same value but the last one. + // This way we are going to have just two time series (and two distinct tsid) and the last dimension identifies + // which time series the document belongs to. + for (int dimId = 0; dimId < numberOfDimensions - 1; dimId++) { + docSource.field("dim_" + dimId, fooDimValue); + } + docSource.field("dim_" + (numberOfDimensions - 1), docId % 2 == 0 ? barDimValue : bazDimValue); + docSource.field("counter_metric", docId + 1); + docSource.field("gauge_metric", randomDoubleBetween(1000.0, 2000.0, true)); + docSource.field("@timestamp", timestampSupplier.get()); + docSource.endObject(); + + return docSource; + } + + private CreateIndexResponse prepareTimeSeriesIndex( + final XContentBuilder mapping, + long startMillis, + long endMillis, + final String[] routingDimensions + ) { + return prepareCreate("index").setSettings( + Settings.builder() + .put("mode", "time_series") + .put("routing_path", String.join(",", routingDimensions)) + .put("index.number_of_shards", randomIntBetween(1, 3)) + .put("index.number_of_replicas", randomIntBetween(1, 3)) + .put("time_series.start_time", startMillis) + .put("time_series.end_time", endMillis) + .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), numberOfDimensions + 1) + .put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), 4192) + .build() + ).setMapping(mapping).get(); + } + + private static Iterator getTimestamps(long startMillis, long endMillis, int numberOfDocs) { + final Set timestamps = new TreeSet<>(); + while (timestamps.size() < numberOfDocs) { + timestamps.add(randomLongBetween(startMillis, endMillis)); + } + return timestamps.iterator(); + } + + private static XContentBuilder timeSeriesIndexMapping() throws IOException { + final XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + builder.startObject("properties"); + for (int i = 0; i < numberOfDimensions; i++) { + builder.startObject("dim_" + i); + builder.field("type", "keyword"); + builder.field("time_series_dimension", true); + builder.endObject(); + } + builder.startObject("counter_metric"); + builder.field("type", "double"); + builder.endObject(); + builder.startObject("gauge_metric"); + builder.field("type", "double"); + builder.endObject(); + builder.endObject(); // properties + builder.endObject(); + return builder; + } + + public void testTimeSeriesAggregation() { + final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts"); + final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); + final InternalTimeSeries ts = (InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0); + assertTimeSeriesAggregation(ts); + } + + public void testSumByTsid() { + final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts").subAggregation( + new SumAggregationBuilder("sum").field("gauge_metric") + ); + final SearchResponse searchResponse = client().prepareSearch("index").setQuery(new MatchAllQueryBuilder()).get(); + assertNotEquals(numberOfDocs, searchResponse.getHits().getHits().length); + final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); + final InternalTimeSeries ts = (InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0); + assertTimeSeriesAggregation(ts); + } + + public void testTermsByTsid() { + final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts").subAggregation( + new TermsAggregationBuilder("terms").field("dim_0") + ); + final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); + final InternalTimeSeries ts = (InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0); + assertTimeSeriesAggregation(ts); + } + + public void testDateHistogramByTsid() { + final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts").subAggregation( + new DateHistogramAggregationBuilder("date_histogram").field("@timestamp").calendarInterval(DateHistogramInterval.MINUTE) + ); + final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); + final InternalTimeSeries ts = (InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0); + assertTimeSeriesAggregation(ts); + } + + public void testCardinalityByTsid() { + final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts").subAggregation( + new CardinalityAggregationBuilder("cardinality").field("dim_" + (numberOfDimensions - 1)) + ); + final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); + final InternalTimeSeries ts = (InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0); + assertTimeSeriesAggregation(ts); + assertTrue( + ts.getBuckets() + .stream() + .map(bucket -> (InternalCardinality) bucket.getAggregations().get("cardinality")) + .map(InternalCardinality::getValue) + .allMatch(value -> value.equals(1L)) + ); + } + + private static void assertTimeSeriesAggregation(final InternalTimeSeries timeSeriesAggregation) { + final List> dimensions = timeSeriesAggregation.getBuckets() + .stream() + .map(InternalTimeSeries.InternalBucket::getKey) + .toList(); + // NOTE: only two time series expected as a result of having just two distinct values for the last dimension + assertEquals(2, dimensions.size()); + + final Map firstTimeSeries = dimensions.get(0); + final Map secondTimeSeries = dimensions.get(1); + + assertTsid(firstTimeSeries); + assertTsid(secondTimeSeries); + } + + private static void assertTsid(final Map timeSeries) { + final Map.Entry tsidEntry = timeSeries.entrySet().stream().toList().get(0); + assertEquals(TimeSeriesIdFieldMapper.NAME, tsidEntry.getKey()); + } } From 7e78dacf8279227941703a7e83455b220787d3ad Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 4 Aug 2023 13:06:11 +0200 Subject: [PATCH 005/125] fix: minor refactor --- ...riesAggregationsUnlimitedDimensionsIT.java | 43 ++++++++++--------- .../index/mapper/TimeSeriesIdFieldMapper.java | 6 +-- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java index aa0dee493beeb..83cdcf8e9ab77 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.aggregations.bucket; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; @@ -29,9 +30,9 @@ import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.InternalCardinality; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; -import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; +import org.junit.Before; import java.io.IOException; import java.util.Iterator; @@ -41,25 +42,25 @@ import java.util.TreeSet; import java.util.function.Supplier; -@ESIntegTestCase.ClusterScope(numDataNodes = 3, numClientNodes = 1, supportsDedicatedMasters = true) -@ESIntegTestCase.SuiteScopeTestCase +import static org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper.TSID_HASH_PREFIX; + public class TimeSeriesAggregationsUnlimitedDimensionsIT extends AggregationIntegTestCase { private static int numberOfDimensions; - private static int numberOfDocs; + private static int numberOfDocuments; - @Override - public void setupSuiteScopeCluster() throws Exception { + @Before + public void setup() throws Exception { numberOfDimensions = randomIntBetween(25, 99); final XContentBuilder mapping = timeSeriesIndexMapping(); long startMillis = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); long endMillis = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-02T00:00:00Z"); - numberOfDocs = randomIntBetween(1000, 2000); - final Iterator timestamps = getTimestamps(startMillis, endMillis, numberOfDocs); + numberOfDocuments = randomIntBetween(1000, 2000); + final Iterator timestamps = getTimestamps(startMillis, endMillis, numberOfDocuments); // NOTE: use also the last (changing) dimension so to make sure documents are not indexed all in the same shard. final String[] routingDimensions = new String[] { "dim_1", "dim_" + (numberOfDimensions - 1) }; assertTrue(prepareTimeSeriesIndex(mapping, startMillis, endMillis, routingDimensions).isAcknowledged()); - logger.info("Dimensions: " + numberOfDimensions + " docs: " + numberOfDocs + " start: " + startMillis + " end: " + endMillis); + logger.info("Dimensions: " + numberOfDimensions + " docs: " + numberOfDocuments + " start: " + startMillis + " end: " + endMillis); // NOTE: we need the tsid to be larger than 32 Kb final String fooDimValue = "foo_" + randomAlphaOfLengthBetween(2048, 2048 + 128); @@ -67,7 +68,7 @@ public void setupSuiteScopeCluster() throws Exception { final String bazDimValue = "baz_" + randomAlphaOfLengthBetween(2048, 2048 + 512); final BulkRequestBuilder bulkIndexRequest = client().prepareBulk(); - for (int docId = 0; docId < numberOfDocs; docId++) { + for (int docId = 0; docId < numberOfDocuments; docId++) { final XContentBuilder document = timeSeriesDocument(fooDimValue, barDimValue, bazDimValue, docId, timestamps::next); bulkIndexRequest.add(client().prepareIndex("index").setOpType(DocWriteRequest.OpType.CREATE).setSource(document)); } @@ -161,7 +162,7 @@ public void testSumByTsid() { new SumAggregationBuilder("sum").field("gauge_metric") ); final SearchResponse searchResponse = client().prepareSearch("index").setQuery(new MatchAllQueryBuilder()).get(); - assertNotEquals(numberOfDocs, searchResponse.getHits().getHits().length); + assertNotEquals(numberOfDocuments, searchResponse.getHits().getHits().length); final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); final InternalTimeSeries ts = (InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0); assertTimeSeriesAggregation(ts); @@ -187,18 +188,13 @@ public void testDateHistogramByTsid() { public void testCardinalityByTsid() { final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts").subAggregation( - new CardinalityAggregationBuilder("cardinality").field("dim_" + (numberOfDimensions - 1)) + new CardinalityAggregationBuilder("dim_n_cardinality").field("dim_" + (numberOfDimensions - 1)) ); final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); - final InternalTimeSeries ts = (InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0); - assertTimeSeriesAggregation(ts); - assertTrue( - ts.getBuckets() - .stream() - .map(bucket -> (InternalCardinality) bucket.getAggregations().get("cardinality")) - .map(InternalCardinality::getValue) - .allMatch(value -> value.equals(1L)) - ); + assertTimeSeriesAggregation(aggregationResponse.getAggregations().get("counter_cardinality")); + ((InternalTimeSeries) aggregationResponse.getAggregations().get("ts")).getBuckets().forEach(bucket -> { + assertCardinality(bucket.getAggregations().get("dim_n_cardinality"), 2); + }); } private static void assertTimeSeriesAggregation(final InternalTimeSeries timeSeriesAggregation) { @@ -219,5 +215,10 @@ private static void assertTimeSeriesAggregation(final InternalTimeSeries timeSer private static void assertTsid(final Map timeSeries) { final Map.Entry tsidEntry = timeSeries.entrySet().stream().toList().get(0); assertEquals(TimeSeriesIdFieldMapper.NAME, tsidEntry.getKey()); + assertTrue(((BytesRef) tsidEntry.getValue()).utf8ToString().startsWith(TSID_HASH_PREFIX)); + } + + private static void assertCardinality(final InternalCardinality cardinalityAggregation, int expectedCardinality) { + assertEquals(expectedCardinality, cardinalityAggregation.getValue()); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 723f12f6496e5..f00d3ea5d12ca 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -63,7 +63,6 @@ public class TimeSeriesIdFieldMapper extends MetadataFieldMapper { * hundred bytes each, but we're being paranoid here. */ public static final int TSID_HASH_SENTINEL = 0xBAADCAFE; - private static final Base64.Encoder TSID_BASE64_ENCODER = Base64.getUrlEncoder().withoutPadding(); public static final String TSID_HASH_PREFIX = "hash-"; @Override @@ -164,7 +163,7 @@ public static Map decodeTsid(StreamInput in) { try { int sizeOrTsidHashSentinel = in.readVInt(); if (sizeOrTsidHashSentinel == TSID_HASH_SENTINEL) { - return Collections.singletonMap(TimeSeriesIdFieldMapper.NAME, in.readBytesRef()); + return Collections.emptyMap(); } Map result = new LinkedHashMap<>(sizeOrTsidHashSentinel); @@ -226,7 +225,8 @@ public BytesReference build() throws IOException { final byte[] buffer = new byte[16]; hashOut.writeVInt(TSID_HASH_SENTINEL); TsidExtractingIdFieldMapper.hashTsid(timeSeriesId.toBytesRef(), buffer); - hashOut.writeBytesRef(new BytesRef(TSID_HASH_PREFIX + TSID_BASE64_ENCODER.encodeToString(buffer))); + final BytesRef encoded = new BytesRef(Base64.getUrlEncoder().withoutPadding().encodeToString(buffer)); + hashOut.writeBytesRef(encoded); return hashOut.bytes(); } } From a9f439b27cd25af976c51ad4b1018369bbc8eda8 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 29 Aug 2023 14:41:02 +0200 Subject: [PATCH 006/125] fix: refactor and disable test --- ...riesAggregationsUnlimitedDimensionsIT.java | 11 ++--- .../index/mapper/TimeSeriesIdFieldMapper.java | 42 +++++++++++-------- .../mapper/TsidExtractingIdFieldMapper.java | 7 ---- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java index 83cdcf8e9ab77..9d0a69f7e45c4 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -8,7 +8,6 @@ package org.elasticsearch.aggregations.bucket; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; @@ -57,7 +56,7 @@ public void setup() throws Exception { numberOfDocuments = randomIntBetween(1000, 2000); final Iterator timestamps = getTimestamps(startMillis, endMillis, numberOfDocuments); // NOTE: use also the last (changing) dimension so to make sure documents are not indexed all in the same shard. - final String[] routingDimensions = new String[] { "dim_1", "dim_" + (numberOfDimensions - 1) }; + final String[] routingDimensions = new String[] { "dim_0", "dim_" + (numberOfDimensions - 1) }; assertTrue(prepareTimeSeriesIndex(mapping, startMillis, endMillis, routingDimensions).isAcknowledged()); logger.info("Dimensions: " + numberOfDimensions + " docs: " + numberOfDocuments + " start: " + startMillis + " end: " + endMillis); @@ -141,9 +140,11 @@ private static XContentBuilder timeSeriesIndexMapping() throws IOException { } builder.startObject("counter_metric"); builder.field("type", "double"); + builder.field("time_series_metric", "counter"); builder.endObject(); builder.startObject("gauge_metric"); builder.field("type", "double"); + builder.field("time_series_metric", "gauge"); builder.endObject(); builder.endObject(); // properties builder.endObject(); @@ -186,14 +187,14 @@ public void testDateHistogramByTsid() { assertTimeSeriesAggregation(ts); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98715") public void testCardinalityByTsid() { final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts").subAggregation( new CardinalityAggregationBuilder("dim_n_cardinality").field("dim_" + (numberOfDimensions - 1)) ); final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); - assertTimeSeriesAggregation(aggregationResponse.getAggregations().get("counter_cardinality")); ((InternalTimeSeries) aggregationResponse.getAggregations().get("ts")).getBuckets().forEach(bucket -> { - assertCardinality(bucket.getAggregations().get("dim_n_cardinality"), 2); + assertCardinality(bucket.getAggregations().get("dim_n_cardinality"), 1); }); } @@ -215,7 +216,7 @@ private static void assertTimeSeriesAggregation(final InternalTimeSeries timeSer private static void assertTsid(final Map timeSeries) { final Map.Entry tsidEntry = timeSeries.entrySet().stream().toList().get(0); assertEquals(TimeSeriesIdFieldMapper.NAME, tsidEntry.getKey()); - assertTrue(((BytesRef) tsidEntry.getValue()).utf8ToString().startsWith(TSID_HASH_PREFIX)); + assertTrue(((String) tsidEntry.getValue()).startsWith(TSID_HASH_PREFIX)); } private static void assertCardinality(final InternalCardinality cardinalityAggregation, int expectedCardinality) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index f00d3ea5d12ca..4217138974e9d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -15,9 +15,11 @@ import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.hash.MurmurHash3; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.fielddata.FieldData; @@ -140,10 +142,26 @@ private TimeSeriesIdFieldMapper() { public void postParse(DocumentParserContext context) throws IOException { assert fieldType().isIndexed() == false; - TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDimensions(); - BytesRef timeSeriesId = timeSeriesIdBuilder.build().toBytesRef(); - context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); - TsidExtractingIdFieldMapper.createField(context, timeSeriesIdBuilder.routingBuilder, timeSeriesId); + final TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDimensions(); + final BytesReference timeSeriesId = timeSeriesIdBuilder.build(); + final BytesReference timeSeriesIdHashOrPlain = timeSeriesId.toBytesRef().length > LIMIT ? hash(timeSeriesId) : timeSeriesId; + context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesIdHashOrPlain.toBytesRef())); + TsidExtractingIdFieldMapper.createField(context, timeSeriesIdBuilder.routingBuilder, timeSeriesId.toBytesRef()); + } + + public static BytesReference hash(final BytesReference timeSeriesId) throws IOException { + try (BytesStreamOutput out = new BytesStreamOutput()) { + final byte[] buffer = new byte[16]; + out.writeVInt(TSID_HASH_SENTINEL); + final BytesRef tsid = timeSeriesId.toBytesRef(); + final MurmurHash3.Hash128 hash = new MurmurHash3.Hash128(); + MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, 0, hash); + ByteUtils.writeLongLE(hash.h1, buffer, 0); + ByteUtils.writeLongLE(hash.h2, buffer, 8); + final BytesRef encoded = new BytesRef(TSID_HASH_PREFIX + Base64.getUrlEncoder().encodeToString(buffer)); + out.writeBytesRef(encoded); + return out.bytes(); + } } @Override @@ -163,7 +181,7 @@ public static Map decodeTsid(StreamInput in) { try { int sizeOrTsidHashSentinel = in.readVInt(); if (sizeOrTsidHashSentinel == TSID_HASH_SENTINEL) { - return Collections.emptyMap(); + return Collections.singletonMap("_tsid", in.readBytesRef().utf8ToString()); } Map result = new LinkedHashMap<>(sizeOrTsidHashSentinel); @@ -218,19 +236,7 @@ public BytesReference build() throws IOException { out.writeBytesRef(fieldName); entry.getValue().writeTo(out); } - final BytesReference timeSeriesId = out.bytes(); - if (timeSeriesId.length() > LIMIT) { - // NOTE: we hash the _tsid only if necessary, which is if the field does not fit Lucene UTF-8 doc values - try (BytesStreamOutput hashOut = new BytesStreamOutput()) { - final byte[] buffer = new byte[16]; - hashOut.writeVInt(TSID_HASH_SENTINEL); - TsidExtractingIdFieldMapper.hashTsid(timeSeriesId.toBytesRef(), buffer); - final BytesRef encoded = new BytesRef(Base64.getUrlEncoder().withoutPadding().encodeToString(buffer)); - hashOut.writeBytesRef(encoded); - return hashOut.bytes(); - } - } - return timeSeriesId; + return out.bytes(); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 3ede5b0256750..64491e3ef9693 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -171,13 +171,6 @@ public static String createId( return id; } - public static void hashTsid(final BytesRef tsid, byte[] buffer) { - final Hash128 hash = new Hash128(); - MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash); - ByteUtils.writeLongLE(hash.h1, buffer, 0); - ByteUtils.writeLongLE(hash.h2, buffer, 8); - } - @Override public String documentDescription(DocumentParserContext context) { /* From e24c3be16928341c3e0c3b8360f62ab57288117f Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 4 Sep 2023 11:21:06 +0200 Subject: [PATCH 007/125] fix: always hash the tsid --- .../elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 4217138974e9d..72c669e8c8dfd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -144,12 +144,11 @@ public void postParse(DocumentParserContext context) throws IOException { final TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDimensions(); final BytesReference timeSeriesId = timeSeriesIdBuilder.build(); - final BytesReference timeSeriesIdHashOrPlain = timeSeriesId.toBytesRef().length > LIMIT ? hash(timeSeriesId) : timeSeriesId; - context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesIdHashOrPlain.toBytesRef())); + context.doc().add(new SortedDocValuesField(fieldType().name(), hash128(timeSeriesId).toBytesRef())); TsidExtractingIdFieldMapper.createField(context, timeSeriesIdBuilder.routingBuilder, timeSeriesId.toBytesRef()); } - public static BytesReference hash(final BytesReference timeSeriesId) throws IOException { + public static BytesReference hash128(final BytesReference timeSeriesId) throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { final byte[] buffer = new byte[16]; out.writeVInt(TSID_HASH_SENTINEL); From 58b44f9f4d2107dddc0afcbb32ea6401a8cff22d Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 4 Sep 2023 12:04:29 +0200 Subject: [PATCH 008/125] fix: base64 without padding --- .../org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 72c669e8c8dfd..99535b72e87c2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -157,7 +157,7 @@ public static BytesReference hash128(final BytesReference timeSeriesId) throws I MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, 0, hash); ByteUtils.writeLongLE(hash.h1, buffer, 0); ByteUtils.writeLongLE(hash.h2, buffer, 8); - final BytesRef encoded = new BytesRef(TSID_HASH_PREFIX + Base64.getUrlEncoder().encodeToString(buffer)); + final BytesRef encoded = new BytesRef(TSID_HASH_PREFIX + Base64.getUrlEncoder().withoutPadding().encodeToString(buffer)); out.writeBytesRef(encoded); return out.bytes(); } From 6a92d61ebe2cf13e39370d4ab4eb985fda971799 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 4 Sep 2023 12:17:52 +0200 Subject: [PATCH 009/125] fix: remove hash prefix --- .../bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java | 3 --- .../resources/rest-api-spec/test/data_stream/150_tsdb.yml | 2 +- .../elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java | 3 +-- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java index 9d0a69f7e45c4..8452594e52714 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -41,8 +41,6 @@ import java.util.TreeSet; import java.util.function.Supplier; -import static org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper.TSID_HASH_PREFIX; - public class TimeSeriesAggregationsUnlimitedDimensionsIT extends AggregationIntegTestCase { private static int numberOfDimensions; private static int numberOfDocuments; @@ -216,7 +214,6 @@ private static void assertTimeSeriesAggregation(final InternalTimeSeries timeSer private static void assertTsid(final Map timeSeries) { final Map.Entry tsidEntry = timeSeries.entrySet().stream().toList().get(0); assertEquals(TimeSeriesIdFieldMapper.NAME, tsidEntry.getKey()); - assertTrue(((String) tsidEntry.getValue()).startsWith(TSID_HASH_PREFIX)); } private static void assertCardinality(final InternalCardinality cardinalityAggregation, int expectedCardinality) { diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index b1b6cfa772bcc..e5f26ee8cf61a 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -143,7 +143,7 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]} + - match: {hits.hits.0.fields._tsid: hash-g11Xnx49I8go3KMV7pQ9_Q} --- "aggregate the tsid": diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 99535b72e87c2..82297a9252781 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -65,7 +65,6 @@ public class TimeSeriesIdFieldMapper extends MetadataFieldMapper { * hundred bytes each, but we're being paranoid here. */ public static final int TSID_HASH_SENTINEL = 0xBAADCAFE; - public static final String TSID_HASH_PREFIX = "hash-"; @Override public FieldMapper.Builder getMergeBuilder() { @@ -157,7 +156,7 @@ public static BytesReference hash128(final BytesReference timeSeriesId) throws I MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, 0, hash); ByteUtils.writeLongLE(hash.h1, buffer, 0); ByteUtils.writeLongLE(hash.h2, buffer, 8); - final BytesRef encoded = new BytesRef(TSID_HASH_PREFIX + Base64.getUrlEncoder().withoutPadding().encodeToString(buffer)); + final BytesRef encoded = new BytesRef(Base64.getUrlEncoder().withoutPadding().encodeToString(buffer)); out.writeBytesRef(encoded); return out.bytes(); } From 56645dbe363bb104fdc11e5d35b2ba99a3ed0e47 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 4 Sep 2023 15:33:06 +0200 Subject: [PATCH 010/125] fix: use tsid hash values in different tests --- .../rest-api-spec/test/aggregations/time_series.yml | 2 +- .../rest-api-spec/test/data_stream/150_tsdb.yml | 6 +++--- .../rest-api-spec/test/tsdb/30_snapshot.yml | 2 +- .../rest-api-spec/test/tsdb/60_add_dimensions.yml | 2 +- .../rest-api-spec/test/tsdb/70_dimension_types.yml | 12 ++++++------ .../rest-api-spec/test/tsdb/80_index_resize.yml | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 553ad5c2019d7..b57ed6b1ab84e 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -76,7 +76,7 @@ setup: - match: { hits.total.value: 1 } - length: { aggregations: 1 } - - match: { aggregations.ts.buckets.0.key: { "key": "foo" } } + - match: { aggregations.ts.buckets.0.key: { _tsid: HItuiNXRFKqL3KHB3Q7OZQ } } - match: { aggregations.ts.buckets.0.doc_count: 1 } --- diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index e5f26ee8cf61a..360d79ce19d1d 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -143,7 +143,7 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: hash-g11Xnx49I8go3KMV7pQ9_Q} + - match: {hits.hits.0.fields._tsid: [{_tsid: g11Xnx49I8go3KMV7pQ9_Q }]} --- "aggregate the tsid": @@ -164,9 +164,9 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: M5S3HK24uRDBXagnqLaSrQ}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: g11Xnx49I8go3KMV7pQ9_Q}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 1e6dd9243fbe3..2981141cc8447 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -148,5 +148,5 @@ teardown: - match: {hits.total.value: 1} - match: {hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507} - - match: {hits.hits.0.fields._tsid: [ { k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod } ] } + - match: {hits.hits.0.fields._tsid: [{ _tsid: g11Xnx49I8go3KMV7pQ9_Q }]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 2c151b9b7139f..3838be7ea0eb5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -96,7 +96,7 @@ add dimensions to no dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { metricset: cat } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: U6txPvtQs7IIPDHGBuj5hg } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 2fb3209f6b845..712e3df809106 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -445,19 +445,19 @@ flattened field misspelled routing path field: - match: { hits.total.value: 6 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: "1516op6778" } + - match: { aggregations.tsids.buckets.0.key._tsid: CGcajru3QohMHSqC7Hamag } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.09, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: "1516op6885" } + - match: { aggregations.tsids.buckets.1.key._tsid: QTTaRfl0gDouwjSzzE09QA } - match: { aggregations.tsids.buckets.1.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.40, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.59, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: "16w3xaca09" } + - match: { aggregations.tsids.buckets.2.key._tsid: aCmczDP-VvnayVco5Uds2A } - match: { aggregations.tsids.buckets.2.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 6.59, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: "16w3xacq34" } + - match: { aggregations.tsids.buckets.3.key._tsid: usXWhQGRpc1D-Uc3-LF2RQ } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index f78f16780cb4f..d9838ee58eab0 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -123,7 +123,7 @@ shrink: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: g11Xnx49I8go3KMV7pQ9_Q}]} --- clone: @@ -147,4 +147,4 @@ clone: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: g11Xnx49I8go3KMV7pQ9_Q}]} From 1364c5923e5d116f77694226752f2ebadbf32de1 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 7 Sep 2023 11:19:18 +0200 Subject: [PATCH 011/125] fix: use similarity hash to hash the _tsid --- .../test/aggregations/time_series.yml | 8 +- .../test/tsdb/60_add_dimensions.yml | 2 +- .../test/tsdb/70_dimension_types.yml | 8 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 80 ++++++++++++++++--- .../mapper/TimeSeriesIdFieldMapperTests.java | 34 ++++---- 5 files changed, 90 insertions(+), 42 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index b57ed6b1ab84e..15b5561c20af8 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -126,7 +126,7 @@ setup: size: 1 - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } + - match: { aggregations.ts.buckets.0.key: { _tsid: HItuiNXRFKqL3KHB3Q7OZQ } } - do: search: @@ -144,9 +144,9 @@ setup: size: 3 - length: { aggregations.ts.buckets: 3 } - - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } - - match: { aggregations.ts.buckets.1.key: { "key": "baz" } } - - match: { aggregations.ts.buckets.2.key: { "key": "foo" } } + - match: { aggregations.ts.buckets.0.key: { _tsid: HItuiNXRFKqL3KHB3Q7OZQ } } + - match: { aggregations.ts.buckets.1.key: { _tsid: fCw9K58Pz_Ibpj8g2Gdtsg } } + - match: { aggregations.ts.buckets.2.key: { _tsid: g4DKHI17Abkfzziime-XEQ } } --- "Score test filter some": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 3838be7ea0eb5..a7f2dfc84a334 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -48,7 +48,7 @@ add dimensions with put_mapping: - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { metricset: cat } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: U6txPvtQs7IIPDHGBuj5hg } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 712e3df809106..4989cdad55ffa 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -604,12 +604,12 @@ ip dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: { ip: "10.10.1.1", metricset: aa}} + - match: {aggregations.tsids.buckets.0.key: { _tsid: QU-IiU8tS9Jt4bFybfAyjA}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 7.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: { ip: "2001:db8:85a3::8a2e:370:7334", metricset: aa }} + - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} + - match: {aggregations.tsids.buckets.1.key: { _tsid: vDOFKD7E2E0acYAihneuTg }} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 3.3, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} --- runtime time series dimension: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 82297a9252781..f362df3cf4b84 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -12,6 +12,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.StringHelper; import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -35,12 +36,14 @@ import java.io.IOException; import java.net.InetAddress; import java.time.ZoneId; +import java.util.Arrays; import java.util.Base64; import java.util.Collections; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; +import java.util.SortedSet; +import java.util.TreeSet; /** * Mapper for {@code _tsid} field included generated when the index is @@ -143,10 +146,11 @@ public void postParse(DocumentParserContext context) throws IOException { final TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDimensions(); final BytesReference timeSeriesId = timeSeriesIdBuilder.build(); - context.doc().add(new SortedDocValuesField(fieldType().name(), hash128(timeSeriesId).toBytesRef())); + context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesIdBuilder.similarityHash(timeSeriesId).toBytesRef())); TsidExtractingIdFieldMapper.createField(context, timeSeriesIdBuilder.routingBuilder, timeSeriesId.toBytesRef()); } + // TODO: remove if using {@link TimeSeriesIdBuilder#similarityHash(BytesReference)} public static BytesReference hash128(final BytesReference timeSeriesId) throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { final byte[] buffer = new byte[16]; @@ -156,6 +160,7 @@ public static BytesReference hash128(final BytesReference timeSeriesId) throws I MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, 0, hash); ByteUtils.writeLongLE(hash.h1, buffer, 0); ByteUtils.writeLongLE(hash.h2, buffer, 8); + // TODO: maybe remove Base64 encoding and do it in {@link TimeSeriesIdFieldMapper#decodeTsid(StreamInput)} )} final BytesRef encoded = new BytesRef(Base64.getUrlEncoder().withoutPadding().encodeToString(buffer)); out.writeBytesRef(encoded); return out.bytes(); @@ -179,7 +184,8 @@ public static Map decodeTsid(StreamInput in) { try { int sizeOrTsidHashSentinel = in.readVInt(); if (sizeOrTsidHashSentinel == TSID_HASH_SENTINEL) { - return Collections.singletonMap("_tsid", in.readBytesRef().utf8ToString()); + final BytesRef bytesRef = in.readBytesRef(); + return Collections.singletonMap("_tsid", Base64.getUrlEncoder().withoutPadding().encodeToString(bytesRef.bytes)); } Map result = new LinkedHashMap<>(sizeOrTsidHashSentinel); @@ -206,12 +212,17 @@ public static Map decodeTsid(StreamInput in) { } public static class TimeSeriesIdBuilder implements DocumentDimensions { + + public static final int MAX_DIMENSIONS = 512; + + private record DimensionDataHolder(BytesReference fieldName, int fieldNameHash, BytesReference value, int valueHash) {} + /** - * A sorted map of the serialized values of dimension fields that will be used + * A sorted set of the serialized values of dimension fields that will be used * for generating the _tsid field. The map will be used by {@link TimeSeriesIdFieldMapper} * to build the _tsid field for the document. */ - private final SortedMap dimensions = new TreeMap<>(); + private final SortedSet dimensions = new TreeSet<>(Comparator.comparing(o -> o.fieldName)); /** * Builds the routing. Used for building {@code _id}. If null then skipped. */ @@ -229,15 +240,51 @@ public BytesReference build() throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(dimensions.size()); - for (Map.Entry entry : dimensions.entrySet()) { - final BytesRef fieldName = entry.getKey(); - out.writeBytesRef(fieldName); - entry.getValue().writeTo(out); + for (DimensionDataHolder entry : dimensions) { + out.writeBytesRef(entry.fieldName.toBytesRef()); + entry.value.writeTo(out); } return out.bytes(); } } + /** + * Here we build the hash of the tsid using a similarity function so that we have a result + * with the following pattern: + * + * ${similarityHash(_tsid)}-${hash(_tsid)}. + * + * The idea is to be able to place 'similar' time series close to each other. Two time series + * are considered 'similar' if they share the same values for a subset of the dimensions (sorted + * names/values). + */ + public BytesReference similarityHash(final BytesReference timeSeriesId) throws IOException { + int bufferIndex = 0; + // 512 entries of (hash32(fieldName) + '=' + hash32(fieldValue) + ":") plus the timeSeriesIdBytesRef hash on 128 bits + final byte[] buffer = new byte[MAX_DIMENSIONS * 10 + 16]; + for (final DimensionDataHolder dimensionDataHolder : dimensions) { + if (bufferIndex >= MAX_DIMENSIONS) break; + ByteUtils.writeIntLE(dimensionDataHolder.fieldNameHash, buffer, bufferIndex); + ByteUtils.writeIntLE('=', buffer, bufferIndex + 4); + ByteUtils.writeIntLE(dimensionDataHolder.valueHash, buffer, bufferIndex + 5); + ByteUtils.writeIntLE(':', buffer, bufferIndex + 9); + bufferIndex += 10; + } + final BytesRef timeSeriesIdBytesRef = timeSeriesId.toBytesRef(); + final MurmurHash3.Hash128 tsidFullHash = new MurmurHash3.Hash128(); + MurmurHash3.hash128(timeSeriesIdBytesRef.bytes, timeSeriesIdBytesRef.offset, timeSeriesIdBytesRef.length, 0, tsidFullHash); + ByteUtils.writeLongLE(tsidFullHash.h1, buffer, bufferIndex); + bufferIndex += 8; + ByteUtils.writeLongLE(tsidFullHash.h2, buffer, bufferIndex); + bufferIndex += 8; + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(TSID_HASH_SENTINEL); + final BytesRef hash = new BytesRef(Arrays.copyOfRange(buffer, 0, bufferIndex)); + out.writeBytesRef(hash); + return out.bytes(); + } + } + @Override public void addString(String fieldName, BytesRef utf8Value) { try (BytesStreamOutput out = new BytesStreamOutput()) { @@ -291,11 +338,18 @@ public void addUnsignedLong(String fieldName, long value) { } } - private void add(String fieldName, BytesReference encoded) { - BytesReference old = dimensions.put(new BytesRef(fieldName), encoded); - if (old != null) { + private void add(String fieldName, BytesReference encoded) throws IOException { + final BytesArray fieldNameBytesRef = new BytesArray(fieldName); + final DimensionDataHolder dimension = new DimensionDataHolder( + fieldNameBytesRef, + StringHelper.murmurhash3_x86_32(fieldNameBytesRef.toBytesRef(), 0), + encoded, + StringHelper.murmurhash3_x86_32(encoded.toBytesRef(), 0) + ); + if (dimensions.contains(dimension)) { throw new IllegalArgumentException("Dimension field [" + fieldName + "] cannot be a multi-valued field."); } + dimensions.add(dimension); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 14dcf6118231c..82d4d8e2acbec 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.List; import java.util.Map; import static org.elasticsearch.test.MapMatcher.assertMap; @@ -77,15 +76,9 @@ public void testEnabledInTimeSeriesMode() throws Exception { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100).field("c", 500)); - assertThat( - doc.rootDoc().getBinaryValue("_tsid"), - equalTo(new BytesRef("\u0002\u0001as\u0005value\u0001bl\u0000\u0000\u0000\u0000\u0000\u0000\u0000d")) - ); - assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); - assertThat(doc.rootDoc().getField("b").numericValue(), equalTo(100L)); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("a", "value").entry("b", 100L) + matchesMap().entry("_tsid", "smklPD3uDjjxOgN-3pU9PwT2Ojp-syegKjheHg5h5ERLJH6q") ); } @@ -132,7 +125,7 @@ public void testStrings() throws IOException { ); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("a", "foo").entry("o.e", "bort") + matchesMap().entry("_tsid", "smklPD2v9houOmeyDcA9tA-WtDo47R9yulyYUoudsXFaYRtH") ); } @@ -148,9 +141,7 @@ public void testUnicodeKeys() throws IOException { Map tsid = TimeSeriesIdFieldMapper.decodeTsid( new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes) ); - assertMap(tsid, matchesMap().entry(coffee, "good").entry(fire, "hot")); - // Also make sure the keys are in order - assertThat(List.copyOf(tsid.keySet()), equalTo(List.of(coffee, fire))); + assertMap(tsid, matchesMap().entry("_tsid", "2vZdwj2ejAXjOviZF9Y9DKlQ6jqs_5V1RcDg-M6kBhZ5cO40")); } public void testKeywordTooLong() throws IOException { @@ -161,7 +152,7 @@ public void testKeywordTooLong() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("a", "more_than_1024_bytes".repeat(52)) + matchesMap().entry("_tsid", "smklPD29ZDUoOsglT7sfXPKk_XyMzgZv1OM") ); } @@ -174,7 +165,7 @@ public void testKeywordTooLongUtf8() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("a", theWordLong.repeat(200)) + matchesMap().entry("_tsid", "smklPD1M43f6OplCG-jkf0ftosS7HTvPX9Q") ); } @@ -215,7 +206,7 @@ public void testLong() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", 1234L) + matchesMap().entry("_tsid", "smklPD2tm41vOuD2FrQ9SrpJyzpnsg3APd1wa2A6wmulw_SDY7Ceqg2_cpDqoQ") ); } @@ -269,7 +260,7 @@ public void testInteger() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Integer.MIN_VALUE) + matchesMap().entry("_tsid", "smklPD2tm41vOuD2FrQ9SrpJyzpnsg3APdXyEVA6iVca07m_g8ACVY2SDNSrWw") ); } @@ -327,7 +318,7 @@ public void testShort() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Short.MIN_VALUE) + matchesMap().entry("_tsid", "smklPD2tm41vOuD2FrQ9SrpJyzpnsg3APSYAWY86NJtZIt97lRcRUmcsL5Wx2Q") ); } @@ -385,7 +376,7 @@ public void testByte() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Byte.MIN_VALUE) + matchesMap().entry("_tsid", "smklPD2tm41vOuD2FrQ9SrpJyzpnsg3APSgrqoc6Jd0t9U26YHnblyf7j8D6Nw") ); } @@ -443,7 +434,7 @@ public void testIp() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("kw", "kw").entry("a", "192.168.0.1").entry("o.e", "255.255.255.1") + matchesMap().entry("_tsid", "smklPD0LrOlWOuD2FrQ9SrpJyzpnsg3APbPzTuw6XwWVVVRBrzAohq4krNO_Vg") ); } @@ -479,7 +470,10 @@ public void testVeryLarge() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", new BytesRef("hash-I0H5PpthdzedQ50vMO9Y3Q")) + matchesMap().entry( + "_tsid", + "A37elT2v9houOt_AxbA9WKlBkzpDxtikPVipQZM63evtJj1YqUGTOmcGd009WKlBkzoL00iOPVipQZM6HcyYDT1YqUGTOmP60TU9WKlBkzomGtdvPVipQZM6nzBSpz1YqUGTOhCUdPc9WKlBkzrt94dlPVipQZM6tpOXzz1YqUGTOh3AKiU9WKlBkzqzeblUPVipQZM64cqTnD1YqUGTOo7laSA9WKlBkzrsUq91PVipQZM6werJIj1YqUGTOv35ZJw9WKlBkzqVPxV2PVipQZM6EXQ--T1YqUGTOrQj-k49WKlBkzpvCEIlPVipQZM6aBOtmD1YqUGTOuPkbfc9WKlBkzpbjZwZPVipQZM69D3Ilz1YqUGTOuxzV_s9WKlBkzrSfQp1PVipQZM6_cSVAD1YqUGTOnC5_2s9WKlBkzrN0KotPVipQZM6YDdNnj1YqUGTOgHxRIM9WKlBkzofqtnuPVipQZM6Ol1y0z1YqUGTOocA5Ws9WKlBkzo_lx9_PVipQZM6vYmcTj1YqUGTOraOdYA9WKlBkzq4XTKzPVipQZM6IAcd1j1YqUGTOmJz9-09WKlBkzo3MbjRPVipQZM63bx19j1YqUGTOuBxVpk9WKlBkzoGxKMZPVipQZM6QrzUnD1YqUGTOvRyFPE9WKlBkzo6syljPVipQZM6tgdVsT1YqUGTOiNB-T6bYXc3nUOdLzDvWN0" + ) ); } From a55021a3ff432253f04b863b6f63a1e20fe8e98a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 7 Sep 2023 11:32:40 +0200 Subject: [PATCH 012/125] fix: enable test and rewrite comment --- .../bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java | 1 - .../elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java index 8452594e52714..d8e38a9affea3 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -185,7 +185,6 @@ public void testDateHistogramByTsid() { assertTimeSeriesAggregation(ts); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/98715") public void testCardinalityByTsid() { final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts").subAggregation( new CardinalityAggregationBuilder("dim_n_cardinality").field("dim_" + (numberOfDimensions - 1)) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index f362df3cf4b84..e92ff8fd601ae 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -251,16 +251,14 @@ public BytesReference build() throws IOException { /** * Here we build the hash of the tsid using a similarity function so that we have a result * with the following pattern: - * * ${similarityHash(_tsid)}-${hash(_tsid)}. - * * The idea is to be able to place 'similar' time series close to each other. Two time series * are considered 'similar' if they share the same values for a subset of the dimensions (sorted * names/values). */ public BytesReference similarityHash(final BytesReference timeSeriesId) throws IOException { int bufferIndex = 0; - // 512 entries of (hash32(fieldName) + '=' + hash32(fieldValue) + ":") plus the timeSeriesIdBytesRef hash on 128 bits + // max 512 entries of (hash32(fieldName) + '=' + hash32(fieldValue) + ":") plus the has128(timeSeriesId) final byte[] buffer = new byte[MAX_DIMENSIONS * 10 + 16]; for (final DimensionDataHolder dimensionDataHolder : dimensions) { if (bufferIndex >= MAX_DIMENSIONS) break; From 2d4a60a5ba87f033b8ffa203b73fa692fa0dd1c9 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 7 Sep 2023 12:03:39 +0200 Subject: [PATCH 013/125] fix: some _tsid hash values --- .../rest-api-spec/test/aggregations/time_series.yml | 8 ++++---- .../resources/rest-api-spec/test/data_stream/150_tsdb.yml | 6 +++--- .../resources/rest-api-spec/test/tsdb/30_snapshot.yml | 2 +- .../resources/rest-api-spec/test/tsdb/50_alias.yml | 6 +++--- .../rest-api-spec/test/tsdb/60_add_dimensions.yml | 2 +- .../resources/rest-api-spec/test/tsdb/80_index_resize.yml | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 15b5561c20af8..7c06c40e6d990 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -126,7 +126,7 @@ setup: size: 1 - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: { _tsid: HItuiNXRFKqL3KHB3Q7OZQ } } + - match: { aggregations.ts.buckets.0.key: { _tsid: "RGqc4j2Kf_6WOnwsPSufD8_yG6Y_INhnbbI" } } - do: search: @@ -144,9 +144,9 @@ setup: size: 3 - length: { aggregations.ts.buckets: 3 } - - match: { aggregations.ts.buckets.0.key: { _tsid: HItuiNXRFKqL3KHB3Q7OZQ } } - - match: { aggregations.ts.buckets.1.key: { _tsid: fCw9K58Pz_Ibpj8g2Gdtsg } } - - match: { aggregations.ts.buckets.2.key: { _tsid: g4DKHI17Abkfzziime-XEQ } } + - match: { aggregations.ts.buckets.0.key: { _tsid: "RGqc4j2Kf_6WOnwsPSufD8_yG6Y_INhnbbI" } } + - match: { aggregations.ts.buckets.1.key: { _tsid: "RGqc4j2v9houOhyLbojV0RSqi9yhwd0OzmU" } } + - match: { aggregations.ts.buckets.2.key: { _tsid: "RGqc4j25AZB2OoOAyhyNewG5H884opnvlxE" } } --- "Score test filter some": diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 360d79ce19d1d..cac3cb649804e 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -143,7 +143,7 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: g11Xnx49I8go3KMV7pQ9_Q }]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39" }]} --- "aggregate the tsid": @@ -164,9 +164,9 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: M5S3HK24uRDBXagnqLaSrQ}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "q2RfQj05ihHDOlaZAWQ9--qoyDozlLccrbi5EMFdqCeotpKt"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: g11Xnx49I8go3KMV7pQ9_Q}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 2981141cc8447..930e4f6e70b7b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -148,5 +148,5 @@ teardown: - match: {hits.total.value: 1} - match: {hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507} - - match: {hits.hits.0.fields._tsid: [{ _tsid: g11Xnx49I8go3KMV7pQ9_Q }]} + - match: {hits.hits.0.fields._tsid: [{ _tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39" }]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index 0035f9be3cfda..c3d77f1f8c3de 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -128,10 +128,10 @@ index into alias: _key: asc - match: {hits.total.value: 12} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "q2RfQj05ihHDOlaZAWQ9--qoyDozlLccrbi5EMFdqCeotpKt"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - match: {aggregations.tsids.buckets.2.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}} + - match: {aggregations.tsids.buckets.2.key: {_tsid: "q2RfQj2-lKWyOlaZAWQ9--qoyDobz8SNhboHMwcV2SSAqY2v"}} - match: {aggregations.tsids.buckets.2.doc_count: 4} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index a7f2dfc84a334..6a5c7d70760c5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -96,7 +96,7 @@ add dimensions to no dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: U6txPvtQs7IIPDHGBuj5hg } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "VpkBZD0GlT_5OlOrcT77ULOyCDwxxgbo-YY" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index d9838ee58eab0..7b0aac87a3cf9 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -147,4 +147,4 @@ clone: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: g11Xnx49I8go3KMV7pQ9_Q}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}]} From 0db1f0d801b787f5aa1b78ae327cfd165f3ff79c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 7 Sep 2023 12:46:31 +0200 Subject: [PATCH 014/125] fix: some more _tsid hash values --- .../resources/rest-api-spec/test/tsdb/40_search.yml | 4 ++-- .../rest-api-spec/test/tsdb/60_add_dimensions.yml | 2 +- .../rest-api-spec/test/tsdb/70_dimension_types.yml | 12 ++++++------ .../rest-api-spec/test/tsdb/80_index_resize.yml | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index 48f650a05d6f9..0bde3fd8cc355 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -281,9 +281,9 @@ aggregate a tag: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "q2RfQj05ihHDOlaZAWQ9--qoyDozlLccrbi5EMFdqCeotpKt"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 6a5c7d70760c5..c831572ae1008 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -196,7 +196,7 @@ add dimensions to some dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { metricset: cat, other_dim: cat } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "VpkBZD0GlT_5OportLw9BpU_-Tpg0xF7LEdq7PSCPmOfbe2l"} ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 4989cdad55ffa..c012ba32fd333 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -529,12 +529,12 @@ long dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {id: 1, metricset: aa}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "4Yt2mD1DgKYdOlaZAWQ973outjqByH6Leukt8Syv6_LtDjuE"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 7.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {id: 2, metricset: aa }} + - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "4Yt2mD2tm41vOlaZAWQ973outjrbyf07xExY7uwo0Fb0U9i9"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 3.3, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} --- ip dimension: @@ -604,10 +604,10 @@ ip dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: { _tsid: QU-IiU8tS9Jt4bFybfAyjA}} + - match: {aggregations.tsids.buckets.0.key: { _tsid: "4Yt2mD1DgKYdOlaZAWQ973outjqByH6Leukt8Syv6_LtDjuE"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: { _tsid: vDOFKD7E2E0acYAihneuTg }} + - match: {aggregations.tsids.buckets.1.key: { _tsid: "4Yt2mD2tm41vOlaZAWQ973outjrbyf07xExY7uwo0Fb0U9i9" }} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index 7b0aac87a3cf9..f3944cb4c5c95 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -123,7 +123,7 @@ shrink: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: g11Xnx49I8go3KMV7pQ9_Q}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}]} --- clone: From 6bb36954677c7e485e6dee746be3ae80948268be Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 7 Sep 2023 15:10:58 +0200 Subject: [PATCH 015/125] fix: some more _tsid values and yaml test skip version --- .../test/aggregations/time_series.yml | 10 +- .../test/tsdb/25_id_generation.yml | 14 +- .../rest-api-spec/test/tsdb/40_search.yml | 22 +- .../test/tsdb/60_add_dimensions.yml | 26 +-- .../test/tsdb/70_dimension_types.yml | 188 +++++++++--------- 5 files changed, 130 insertions(+), 130 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 7c06c40e6d990..67f648f7e4b74 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -54,8 +54,8 @@ setup: --- "Basic test": - skip: - version: " - 8.6.99" - reason: Time series result serialization changed in 8.6.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: search: @@ -76,7 +76,7 @@ setup: - match: { hits.total.value: 1 } - length: { aggregations: 1 } - - match: { aggregations.ts.buckets.0.key: { _tsid: HItuiNXRFKqL3KHB3Q7OZQ } } + - match: { aggregations.ts.buckets.0.key: { _tsid: "RGqc4j2v9houOhyLbojV0RSqi9yhwd0OzmU" } } - match: { aggregations.ts.buckets.0.doc_count: 1 } --- @@ -107,8 +107,8 @@ setup: --- "Size test": - skip: - version: " - 8.6.99" - reason: Size added in 8.7.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index cbf11c0b1d250..8f109eb2ca957 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -211,8 +211,8 @@ create operation on top of old document fails: --- create operation on top of old document fails over bulk: - skip: - version: " - 8.1.99" - reason: id generation changed in 8.2 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: bulk: @@ -221,13 +221,13 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[cn4exTOUtxytuLkQAAABeRnR_mY][{k8s.pod.uid=df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset=pod}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[cn4exTOUtxytuLkQAAABeRnR_mY][{_tsid=q2RfQj05ihHDOlaZAWQ9--qoyDozlLccrbi5EMFdqCeotpKt}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: - skip: - version: " - 8.1.99" - reason: ids generation changed in 8.2 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: search: @@ -240,9 +240,9 @@ ids query: values: ["cn4exTOUtxytuLkQAAABeRnR_mY", "cZZNs4NdV58ePSPIAAABeRnSA5M"] sort: ["@timestamp"] - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "cn4exTOUtxytuLkQAAABeRnR_mY"} + - match: {hits.hits.0._id: "cn4exWs7v3M2l8W_AAABeRnR_mY"} - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434595272]} - - match: {hits.hits.1._id: "cZZNs4NdV58ePSPIAAABeRnSA5M"} + - match: {hits.hits.1._id: "cZZNs-e5_7Eqw0RIAAABeRnSA5M"} - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [2012916202]} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index 0bde3fd8cc355..9154e2068214f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -170,8 +170,8 @@ fetch a tag: --- "fetch the tsid": - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: search: @@ -184,7 +184,7 @@ fetch a tag: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}]} --- aggregate a dimension: @@ -265,8 +265,8 @@ aggregate a tag: --- "aggregate the tsid": - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: search: @@ -308,8 +308,8 @@ aggregate a tag: --- sort by tsid: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: search: @@ -318,7 +318,7 @@ sort by tsid: sort: [ "_tsid", "@timestamp" ] - match: {hits.total.value: 8} - - match: {hits.hits.0.sort: [{ "k8s.pod.uid" : "947e4ced-1786-4e53-9e0c-5c447e959507", "metricset" : "pod"}, 1619635804467]} - - match: {hits.hits.1.sort: [{ "k8s.pod.uid" : "947e4ced-1786-4e53-9e0c-5c447e959507", "metricset" : "pod"}, 1619635824467]} - - match: {hits.hits.4.sort: [{ "k8s.pod.uid" : "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "metricset" : "pod"}, 1619635803142]} - - match: {hits.hits.7.sort: [{ "k8s.pod.uid" : "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "metricset" : "pod"}, 1619635863142]} + - match: {hits.hits.0.sort: [{ _tsid : "q2RfQj05ihHDOlaZAWQ9--qoyDozlLccrbi5EMFdqCeotpKt"}, 1619635803142]} + - match: {hits.hits.1.sort: [{ _tsid : "q2RfQj05ihHDOlaZAWQ9--qoyDozlLccrbi5EMFdqCeotpKt"}, 1619635823142]} + - match: {hits.hits.4.sort: [{ _tsid : "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}, 1619635804467]} + - match: {hits.hits.7.sort: [{ _tsid : "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}, 1619635864467]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index c831572ae1008..a17a2afc2f871 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -1,8 +1,8 @@ --- add dimensions with put_mapping: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: indices.create: @@ -48,14 +48,14 @@ add dimensions with put_mapping: - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: U6txPvtQs7IIPDHGBuj5hg } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "VpkBZD0GlT_5OlOrcT77ULOyCDwxxgbo-YY" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- add dimensions to no dims with dynamic_template over index: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: indices.create: @@ -102,8 +102,8 @@ add dimensions to no dims with dynamic_template over index: --- add dimensions to no dims with dynamic_template over bulk: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: indices.create: @@ -144,14 +144,14 @@ add dimensions to no dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { metricset: cat } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "VpkBZD0GlT_5OlOrcT77ULOyCDwxxgbo-YY" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- add dimensions to some dims with dynamic_template over index: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: indices.create: @@ -202,8 +202,8 @@ add dimensions to some dims with dynamic_template over index: --- add dimensions to some dims with dynamic_template over bulk: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: indices.create: @@ -247,5 +247,5 @@ add dimensions to some dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { metricset: cat, other_dim: cat } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "VpkBZD0GlT_5OportLw9BpU_-Tpg0xF7LEdq7PSCPmOfbe2l" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index c012ba32fd333..43aa222388a2f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -1,8 +1,8 @@ keyword dimension: - skip: features: close_to - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: @@ -64,19 +64,19 @@ keyword dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {uid: 947e4ced-1786-4e53-9e0c-5c447e959507}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "AIyzSj05ihHDOqC4_tr4552crJx0TqjcW7g"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 7.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9}} + - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "AIyzSj112oDhOtNCfOZzwsTk1dFwRk7N1Ok"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 3.3, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} --- flattened dimension: - skip: features: close_to - version: " - 8.7.99" - reason: flattened field support as dimension added in 8.8.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: @@ -143,44 +143,44 @@ flattened dimension: - match: { hits.total.value: 8} - length: { aggregations.tsids.buckets: 4} - - match: { aggregations.tsids.buckets.0.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: "1516op6778" } - - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.0.key.deployment\.version\.minor: "8" } - - match: { aggregations.tsids.buckets.0.key.deployment\.version\.patch: "0" } + - match: { aggregations.tsids.buckets.0.key._tsid: "_kNKbD0oaZ29OgfsaEw9eRDHRzrIzN4RPXkQx0c6MyW1mT3jCuTKOgCMs0o9ddqA4ToLLl9hH3dYvJfQ2itG5erT" } + - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } + - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.0.key.deployment\.version\.minor: null } + - match: { aggregations.tsids.buckets.0.key.deployment\.version\.patch: null } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.09, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: "1516op6885" } - - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.1.key.deployment\.version\.minor: "8" } - - match: { aggregations.tsids.buckets.1.key.deployment\.version\.patch: "0" } + - match: { aggregations.tsids.buckets.1.key._tsid: "_kNKbD2rnf7qOgfsaEw9eRDHRzrIzN4RPXkQx0c6MyW1mT2LDvTuOgCMs0o9OYoRwzp_JH2u_f61eKTtc7WKDxz2" } + - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } + - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.1.key.deployment\.version\.minor: null } + - match: { aggregations.tsids.buckets.1.key.deployment\.version\.patch: null } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.35, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: "16w3xaca09" } - - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.2.key.deployment\.version\.minor: "8" } - - match: { aggregations.tsids.buckets.2.key.deployment\.version\.patch: "1" } + - match: { aggregations.tsids.buckets.2.key._tsid: "_kNKbD3P1l1UOgfsaEw9eRDHRzrIzN4RPXkQx0c6MyW1mT3jCuTKOgCMs0o9ddqA4TqthmN6Hp0it0rJEKT-tx88" } + - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: null } + - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.2.key.deployment\.version\.minor: null } + - match: { aggregations.tsids.buckets.2.key.deployment\.version\.patch: null } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 6.64, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: "16w3xacq34" } - - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.3.key.deployment\.version\.minor: "8" } - - match: { aggregations.tsids.buckets.3.key.deployment\.version\.patch: "1" } + - match: { aggregations.tsids.buckets.3.key._tsid: "_kNKbD3gKelzOgfsaEw9eRDHRzrIzN4RPXkQx0c6MyW1mT2LDvTuOgCMs0o9OYoRwzq2_k34oJmdL2fZQ7k7CHYf" } + - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: null } + - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.3.key.deployment\.version\.minor: null } + - match: { aggregations.tsids.buckets.3.key.deployment\.version\.patch: null } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.65, error: 0.01 }} --- flattened empty dimension: - skip: features: close_to - version: " - 8.7.99" - reason: flattened field support as dimension added in 8.8.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: @@ -247,21 +247,21 @@ flattened empty dimension: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 2 } - - match: { aggregations.tsids.buckets.0.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } + - match: { aggregations.tsids.buckets.0.key._tsid: "AIyzSj05ihHDOqC4_tr4552crJx0TqjcW7g" } - match: { aggregations.tsids.buckets.0.doc_count: 4 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.22, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } + - match: { aggregations.tsids.buckets.1.key._tsid: "AIyzSj112oDhOtNCfOZzwsTk1dFwRk7N1Ok" } - match: { aggregations.tsids.buckets.1.doc_count: 4 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.69, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.22, error: 0.01 }} --- flattened field missing routing path field: - skip: features: close_to - version: " - 8.7.99" - reason: flattened field support as dimension added in 8.8.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: indices.create: @@ -327,54 +327,54 @@ flattened field missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 6 } - - match: { aggregations.tsids.buckets.0.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } + - match: { aggregations.tsids.buckets.0.key._tsid: "44GrMT3xTXebOgfsaEw9eRDHRzoAjLNKPTmKEcM6weBqMfSkCLURxvYg7_XhDA" } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.0.key.deployment\.build\.branch: "release-8.8" } - - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: "8" } + - match: { aggregations.tsids.buckets.0.key.deployment\.build\.branch: null } + - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.30, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } + - match: { aggregations.tsids.buckets.1.key._tsid: "44GrMT3xTXebOgfsaEw9eRDHRzoAjLNKPXXagOE6GbruwU-xzOsh9ZZJhOBfUw" } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.1.key.deployment\.build\.branch: "release-8.8" } - - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: "8" } + - match: { aggregations.tsids.buckets.1.key.deployment\.build\.branch: null } + - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.1.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.69, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: "1516op6778" } - - match: { aggregations.tsids.buckets.2.key.deployment\.build\.branch: "release-8.8" } - - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.2.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.09, error: 0.01 }} + - match: { aggregations.tsids.buckets.2.key._tsid: "44GrMT3xTXebOv5DSmw9KGmdvToH7GhMPXkQx0c6AIyzSj112oDhOjpbv9RozLKOlpbIO-C6F2M" } + - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: null } + - match: { aggregations.tsids.buckets.2.key.deployment\.build\.branch: null } + - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.2.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: "1516op6885" } - - match: { aggregations.tsids.buckets.3.key.deployment\.build\.branch: "release-8.8" } - - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.3.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 7.40, error: 0.01 }} - - - match: { aggregations.tsids.buckets.4.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.4.key.deployment\.build\.tag: "16w3xaca09" } - - match: { aggregations.tsids.buckets.4.key.deployment\.build\.branch: "release-8.8" } - - match: { aggregations.tsids.buckets.4.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.4.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 6.59, error: 0.01 }} - - - match: { aggregations.tsids.buckets.5.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.5.key.deployment\.build\.tag: "16w3xacq34" } - - match: { aggregations.tsids.buckets.5.key.deployment\.build\.branch: "release-8.8" } - - match: { aggregations.tsids.buckets.5.key.deployment\.version\.major: "8" } - - match: { aggregations.tsids.buckets.5.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.5.voltage.value: { value: 6.75, error: 0.01 }} + - match: { aggregations.tsids.buckets.3.key._tsid: "44GrMT3xTXebOv5DSmw9q53-6joH7GhMPXkQx0c6AIyzSj05ihHDOkZFQrdV9cMHIc2Yv5avEAQ" } + - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: null } + - match: { aggregations.tsids.buckets.3.key.deployment\.build\.branch: null } + - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.3.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} + + - match: { aggregations.tsids.buckets.4.key._tsid: "44GrMT3xTXebOv5DSmw9z9ZdVDoH7GhMPXkQx0c6AIyzSj112oDhOktc_H930xsM7SVUdyQEec0" } + - match: { aggregations.tsids.buckets.4.key.deployment\.build\.tag: null } + - match: { aggregations.tsids.buckets.4.key.deployment\.build\.branch: null } + - match: { aggregations.tsids.buckets.4.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.4.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 7.10, error: 0.01 }} + + - match: { aggregations.tsids.buckets.5.key._tsid: "44GrMT3xTXebOv5DSmw94CnpczoH7GhMPXkQx0c6AIyzSj05ihHDOphogYqrNlaBjWszNoYt_vM" } + - match: { aggregations.tsids.buckets.5.key.deployment\.build\.tag: null } + - match: { aggregations.tsids.buckets.5.key.deployment\.build\.branch: null } + - match: { aggregations.tsids.buckets.5.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.5.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.5.voltage.value: { value: 6.60, error: 0.01 }} --- flattened field misspelled routing path field: - skip: features: close_to - version: " - 8.7.99" - reason: flattened field support as dimension added in 8.8.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: indices.create: @@ -445,28 +445,28 @@ flattened field misspelled routing path field: - match: { hits.total.value: 6 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key._tsid: CGcajru3QohMHSqC7Hamag } - - match: { aggregations.tsids.buckets.0.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.09, error: 0.01 }} + - match: { aggregations.tsids.buckets.0.key._tsid: "_kNKbD0oaZ29OmgpnMwz_lb52slXKOVHbNg" } + - match: { aggregations.tsids.buckets.0.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: QTTaRfl0gDouwjSzzE09QA } - - match: { aggregations.tsids.buckets.1.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.59, error: 0.01 }} + - match: { aggregations.tsids.buckets.1.key._tsid: "_kNKbD2rnf7qOrrF1oUBkaXNQ_lHN_ixdkU" } + - match: { aggregations.tsids.buckets.1.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: aCmczDP-VvnayVco5Uds2A } - - match: { aggregations.tsids.buckets.2.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.40, error: 0.01 }} + - match: { aggregations.tsids.buckets.2.key._tsid: "_kNKbD3P1l1UOghnGo67t0KITB0qgux2pmo" } + - match: { aggregations.tsids.buckets.2.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: usXWhQGRpc1D-Uc3-LF2RQ } - - match: { aggregations.tsids.buckets.3.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} + - match: { aggregations.tsids.buckets.3.key._tsid: "_kNKbD3gKelzOkE02kX5dIA6LsI0s8xNPUA" } + - match: { aggregations.tsids.buckets.3.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} --- long dimension: - skip: features: close_to - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: indices.create: @@ -540,8 +540,8 @@ long dimension: ip dimension: - skip: features: close_to - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.10.99" + reason: _tsid hasing introduced in 8.11 - do: indices.create: @@ -604,10 +604,10 @@ ip dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: { _tsid: "4Yt2mD1DgKYdOlaZAWQ973outjqByH6Leukt8Syv6_LtDjuE"}} + - match: {aggregations.tsids.buckets.0.key: { _tsid: "4h8Hlj3Yzb-vOlaZAWQ973outjpBT4iJTy1L0m3hsXJt8DKM"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: { _tsid: "4Yt2mD2tm41vOlaZAWQ973outjrbyf07xExY7uwo0Fb0U9i9" }} + - match: {aggregations.tsids.buckets.1.key: { _tsid: "4h8Hlj3aaOFKOlaZAWQ973outjq8M4UoPsTYTRpxgCKGd65O"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} From a435a066a110063e492aae30ceee48c8632e2b87 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 5 Oct 2023 15:03:13 +0200 Subject: [PATCH 016/125] fix: increase the dimension field limit At some point we might remove it completely. --- .../main/java/org/elasticsearch/index/mapper/MapperService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 512c35a146d0c..1b6f1e760fd87 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -113,7 +113,7 @@ public enum MergeReason { ); public static final Setting INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING = Setting.longSetting( "index.mapping.dimension_fields.limit", - 21, + 9999, 0, Property.Dynamic, Property.IndexScope From fffc5aa35d74f2c69099d972e91da499323bc828 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 17 Oct 2023 16:00:52 +0200 Subject: [PATCH 017/125] fix: hash the tsid field using a new hashing scheme --- .../timeseries/InternalTimeSeriesTests.java | 2 +- .../timeseries/TimeSeriesAggregatorTests.java | 4 +- .../test/tsdb/25_id_generation.yml | 6 +- .../rest-api-spec/test/tsdb/30_snapshot.yml | 2 +- .../rest-api-spec/test/tsdb/40_search.yml | 14 +- .../rest-api-spec/test/tsdb/50_alias.yml | 10 +- .../test/tsdb/60_add_dimensions.yml | 10 +- .../test/tsdb/70_dimension_types.yml | 44 ++--- .../test/tsdb/80_index_resize.yml | 4 +- .../support/TimeSeriesDimensionsLimitIT.java | 3 +- .../org/elasticsearch/index/IndexMode.java | 10 +- ...entDimensions.java => DocumentFields.java} | 35 ++-- .../index/mapper/DocumentParserContext.java | 14 +- .../index/mapper/IpFieldMapper.java | 2 +- .../index/mapper/KeywordFieldMapper.java | 2 +- .../index/mapper/NumberFieldMapper.java | 4 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 176 +++++++++++------- .../flattened/FlattenedFieldParser.java | 2 +- .../elasticsearch/search/DocValueFormat.java | 8 +- .../index/mapper/IdLoaderTests.java | 8 +- .../mapper/TimeSeriesIdFieldMapperTests.java | 90 +++++++-- .../search/DocValueFormatTests.java | 6 +- .../rate/TimeSeriesRateAggregatorTests.java | 2 +- .../unsignedlong/UnsignedLongFieldMapper.java | 4 +- .../aggregations/GeoLineAggregatorTests.java | 2 +- 25 files changed, 288 insertions(+), 176 deletions(-) rename server/src/main/java/org/elasticsearch/index/mapper/{DocumentDimensions.java => DocumentFields.java} (63%) diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java index 1ed6fa058e643..a9e22010fb164 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java @@ -50,7 +50,7 @@ private List randomBuckets(boolean keyed, InternalAggregations a long docCount = randomLongBetween(0, Long.MAX_VALUE / (20L * numberOfBuckets)); var builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); for (var entry : keys.get(j).entrySet()) { - builder.addString(entry.getKey(), (String) entry.getValue()); + builder.addKeywordDimension(entry.getKey(), (String) entry.getValue()); } try { var key = builder.build().toBytesRef(); diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index 8a3a061750a87..49852331c2b54 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -87,9 +87,9 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens final TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(null); for (int i = 0; i < dimensions.length; i += 2) { if (dimensions[i + 1] instanceof Number n) { - builder.addLong(dimensions[i].toString(), n.longValue()); + builder.addLongDimension(dimensions[i].toString(), n.longValue()); } else { - builder.addString(dimensions[i].toString(), dimensions[i + 1].toString()); + builder.addKeywordDimension(dimensions[i].toString(), dimensions[i + 1].toString()); } } for (int i = 0; i < metrics.length; i += 2) { diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 8f109eb2ca957..f54b9563d4b2d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -221,7 +221,7 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[cn4exTOUtxytuLkQAAABeRnR_mY][{_tsid=q2RfQj05ihHDOlaZAWQ9--qoyDozlLccrbi5EMFdqCeotpKt}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[cn4exTOUtxytuLkQAAABeRnR_mY][{_tsid=_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyP6Vt9ULEHW_gNYfd2t0r5Rnf6-L1KQ}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: @@ -240,9 +240,9 @@ ids query: values: ["cn4exTOUtxytuLkQAAABeRnR_mY", "cZZNs4NdV58ePSPIAAABeRnSA5M"] sort: ["@timestamp"] - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "cn4exWs7v3M2l8W_AAABeRnR_mY"} + - match: {hits.hits.0._id: "cn4exWfgWRhamteRAAABeRnR_mY"} - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434595272]} - - match: {hits.hits.1._id: "cZZNs-e5_7Eqw0RIAAABeRnSA5M"} + - match: {hits.hits.1._id: "cZZNs6__M5OgAaQpAAABeRnSA5M"} - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [2012916202]} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 930e4f6e70b7b..a8ba6a6a6fd18 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -148,5 +148,5 @@ teardown: - match: {hits.total.value: 1} - match: {hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507} - - match: {hits.hits.0.fields._tsid: [{ _tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39" }]} + - match: {hits.hits.0.fields._tsid: [{ _tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g" }]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index 9154e2068214f..69ac13d66ef14 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -184,7 +184,7 @@ fetch a tag: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}]} --- aggregate a dimension: @@ -281,9 +281,9 @@ aggregate a tag: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "q2RfQj05ihHDOlaZAWQ9--qoyDozlLccrbi5EMFdqCeotpKt"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyP6Vt9ULEHW_gNYfd2t0r5Rnf6-L1KQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -318,7 +318,7 @@ sort by tsid: sort: [ "_tsid", "@timestamp" ] - match: {hits.total.value: 8} - - match: {hits.hits.0.sort: [{ _tsid : "q2RfQj05ihHDOlaZAWQ9--qoyDozlLccrbi5EMFdqCeotpKt"}, 1619635803142]} - - match: {hits.hits.1.sort: [{ _tsid : "q2RfQj05ihHDOlaZAWQ9--qoyDozlLccrbi5EMFdqCeotpKt"}, 1619635823142]} - - match: {hits.hits.4.sort: [{ _tsid : "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}, 1619635804467]} - - match: {hits.hits.7.sort: [{ _tsid : "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}, 1619635864467]} + - match: {hits.hits.0.sort: [{ _tsid : "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyP6Vt9ULEHW_gNYfd2t0r5Rnf6-L1KQ"}, 1619635803142]} + - match: {hits.hits.1.sort: [{ _tsid : "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyP6Vt9ULEHW_gNYfd2t0r5Rnf6-L1KQ"}, 1619635823142]} + - match: {hits.hits.4.sort: [{ _tsid : "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}, 1619635804467]} + - match: {hits.hits.7.sort: [{ _tsid : "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}, 1619635864467]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index c3d77f1f8c3de..318441b35a2d7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -84,9 +84,9 @@ search an alias: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyP6Vt9ULEHW_gNYfd2t0r5Rnf6-L1KQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset: pod}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -128,10 +128,10 @@ index into alias: _key: asc - match: {hits.total.value: 12} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "q2RfQj05ihHDOlaZAWQ9--qoyDozlLccrbi5EMFdqCeotpKt"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyP6Vt9ULEHW_gNYfd2t0r5Rnf6-L1KQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - match: {aggregations.tsids.buckets.2.key: {_tsid: "q2RfQj2-lKWyOlaZAWQ9--qoyDobz8SNhboHMwcV2SSAqY2v"}} + - match: {aggregations.tsids.buckets.2.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAC-lKWy--qoyP6Vt9ULEMwq1RsoCo9dtDZzvUhI5TA"}} - match: {aggregations.tsids.buckets.2.doc_count: 4} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index a17a2afc2f871..17802979b16b8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -48,7 +48,7 @@ add dimensions with put_mapping: - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "VpkBZD0GlT_5OlOrcT77ULOyCDwxxgbo-YY" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "_pW31QsQ95YnAT0C6QOgFs_zuty0ZP6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAGlT_5_pW31QsQlhR27Th3c0p1ZhIq1uAyqA" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -96,7 +96,7 @@ add dimensions to no dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "VpkBZD0GlT_5OlOrcT77ULOyCDwxxgbo-YY" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "_pW31QsQ95YnAT0C6QOgFs_zuty0ZP6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAGlT_5_pW31QsQlhR27Th3c0p1ZhIq1uAyqA" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -144,7 +144,7 @@ add dimensions to no dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "VpkBZD0GlT_5OlOrcT77ULOyCDwxxgbo-YY" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "_pW31QsQ95YnAT0C6QOgFs_zuty0ZP6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAGlT_5_pW31QsQlhR27Th3c0p1ZhIq1uAyqA" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -196,7 +196,7 @@ add dimensions to some dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "VpkBZD0GlT_5OportLw9BpU_-Tpg0xF7LEdq7PSCPmOfbe2l"} ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "_pW31QsQxhgWVhgKGfmIwHt-7D2B_v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-f6Vt9ULEHp7bXIK5rruMAWHvhNmhYw"} ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -247,5 +247,5 @@ add dimensions to some dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "VpkBZD0GlT_5OportLw9BpU_-Tpg0xF7LEdq7PSCPmOfbe2l" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "_pW31QsQxhgWVhgKGfmIwHt-7D2B_v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-f6Vt9ULEHp7bXIK5rruMAWHvhNmhYw" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 43aa222388a2f..4f554816f5b44 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -64,10 +64,10 @@ keyword dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "AIyzSj05ihHDOqC4_tr4552crJx0TqjcW7g"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "_pW31QsQDh_r6QMn0c6jJhFGYOwd6P6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD_pW31QsQmAXfxM3_zbMMRzVn-ZChwA"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "AIyzSj112oDhOtNCfOZzwsTk1dFwRk7N1Ok"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "_pW31QsQDh_r6QMn0c6jJhFGYOwd6P6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh_pW31QsQVMfWLO-N2j8tEM_EPwLCbQ"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -143,7 +143,7 @@ flattened dimension: - match: { hits.total.value: 8} - length: { aggregations.tsids.buckets: 4} - - match: { aggregations.tsids.buckets.0.key._tsid: "_kNKbD0oaZ29OgfsaEw9eRDHRzrIzN4RPXkQx0c6MyW1mT3jCuTKOgCMs0o9ddqA4ToLLl9hH3dYvJfQ2itG5erT" } + - match: { aggregations.tsids.buckets.0.key._tsid: "_pW31QsQP5PJzRH8-Gbp6mqjByZOmv6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAoaZ29eRDHR3kQx0fjCuTKddqA4f6Vt9ULEAfDLRt1ZH3XxmzpI4VE1d8" } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.0.key.deployment\.version\.minor: null } @@ -151,7 +151,7 @@ flattened dimension: - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "_kNKbD2rnf7qOgfsaEw9eRDHRzrIzN4RPXkQx0c6MyW1mT2LDvTuOgCMs0o9OYoRwzp_JH2u_f61eKTtc7WKDxz2" } + - match: { aggregations.tsids.buckets.1.key._tsid: "_pW31QsQP5PJzRH8-Gbp6mqjByZOmv6Vt9ULEAAAAAAAAAAAAAAAAAAAAACrnf7qeRDHR3kQx0eLDvTuOYoRw_6Vt9ULEBX9MyeUknhbUqQelj0fVGI" } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.1.key.deployment\.version\.minor: null } @@ -159,7 +159,7 @@ flattened dimension: - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "_kNKbD3P1l1UOgfsaEw9eRDHRzrIzN4RPXkQx0c6MyW1mT3jCuTKOgCMs0o9ddqA4TqthmN6Hp0it0rJEKT-tx88" } + - match: { aggregations.tsids.buckets.2.key._tsid: "_pW31QsQP5PJzRH8-Gbp6mqjByZOmv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADP1l1UeRDHR3kQx0fjCuTKddqA4f6Vt9ULEJbbw13wErAIwoyRvfPwBO0" } - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.2.key.deployment\.version\.minor: null } @@ -167,7 +167,7 @@ flattened dimension: - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "_kNKbD3gKelzOgfsaEw9eRDHRzrIzN4RPXkQx0c6MyW1mT2LDvTuOgCMs0o9OYoRwzq2_k34oJmdL2fZQ7k7CHYf" } + - match: { aggregations.tsids.buckets.3.key._tsid: "_pW31QsQP5PJzRH8-Gbp6mqjByZOmv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADgKelzeRDHR3kQx0eLDvTuOYoRw_6Vt9ULECy-cFCrvRwidF9_OFLsEgY" } - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.3.key.deployment\.version\.minor: null } @@ -247,11 +247,11 @@ flattened empty dimension: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 2 } - - match: { aggregations.tsids.buckets.0.key._tsid: "AIyzSj05ihHDOqC4_tr4552crJx0TqjcW7g" } + - match: { aggregations.tsids.buckets.0.key._tsid: "_pW31QsQDh_r6QMn0c6jJhFGYOwd6P6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD_pW31QsQmAXfxM3_zbMMRzVn-ZChwA" } - match: { aggregations.tsids.buckets.0.doc_count: 4 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "AIyzSj112oDhOtNCfOZzwsTk1dFwRk7N1Ok" } + - match: { aggregations.tsids.buckets.1.key._tsid: "_pW31QsQDh_r6QMn0c6jJhFGYOwd6P6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh_pW31QsQVMfWLO-N2j8tEM_EPwLCbQ" } - match: { aggregations.tsids.buckets.1.doc_count: 4 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.22, error: 0.01 }} @@ -327,42 +327,42 @@ flattened field missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 6 } - - match: { aggregations.tsids.buckets.0.key._tsid: "44GrMT3xTXebOgfsaEw9eRDHRzoAjLNKPTmKEcM6weBqMfSkCLURxvYg7_XhDA" } + - match: { aggregations.tsids.buckets.0.key._tsid: "_pW31QsQs0XCB0MdYlNxY9PEJqOWhv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADxTXebeRDHRzmKEcP-lbfVCxDP-9PVa-RfUDC6eRn9DO3J" } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "44GrMT3xTXebOgfsaEw9eRDHRzoAjLNKPXXagOE6GbruwU-xzOsh9ZZJhOBfUw" } + - match: { aggregations.tsids.buckets.1.key._tsid: "_pW31QsQs0XCB0MdYlNxY9PEJqOWhv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADxTXebeRDHR3XagOH-lbfVCxAZbEZUQPevXNiuufZwN2d1" } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.1.doc_count: 1 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "44GrMT3xTXebOv5DSmw9KGmdvToH7GhMPXkQx0c6AIyzSj112oDhOjpbv9RozLKOlpbIO-C6F2M" } + - match: { aggregations.tsids.buckets.2.key._tsid: "_pW31QsQWPttUWoKR4uM6b9jH-_EXv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADxTXebKGmdvXkQx0d12oDh_pW31QsQ1FENdo5JPhSC9XstiiFMZw" } - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.2.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.2.doc_count: 1 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "44GrMT3xTXebOv5DSmw9q53-6joH7GhMPXkQx0c6AIyzSj05ihHDOkZFQrdV9cMHIc2Yv5avEAQ" } + - match: { aggregations.tsids.buckets.3.key._tsid: "_pW31QsQWPttUWoKR4uM6b9jH-_EXv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADxTXebq53-6nkQx0c5ihHD_pW31QsQi5nFHahYKUKfgq8xHVO8cA" } - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.3.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.4.key._tsid: "44GrMT3xTXebOv5DSmw9z9ZdVDoH7GhMPXkQx0c6AIyzSj112oDhOktc_H930xsM7SVUdyQEec0" } + - match: { aggregations.tsids.buckets.4.key._tsid: "_pW31QsQWPttUWoKR4uM6b9jH-_EXv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADxTXebz9ZdVHkQx0d12oDh_pW31QsQ-nmBYp5rUkkaiyO1h0pD9Q" } - match: { aggregations.tsids.buckets.4.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.4.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.4.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.4.doc_count: 2 } - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.5.key._tsid: "44GrMT3xTXebOv5DSmw94CnpczoH7GhMPXkQx0c6AIyzSj05ihHDOphogYqrNlaBjWszNoYt_vM" } + - match: { aggregations.tsids.buckets.5.key._tsid: "_pW31QsQWPttUWoKR4uM6b9jH-_EXv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADxTXeb4Cnpc3kQx0c5ihHD_pW31QsQRLXz7VAY03chH7T4JK6g9g" } - match: { aggregations.tsids.buckets.5.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.5.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.5.key.deployment\.version\.major: null } @@ -445,19 +445,19 @@ flattened field misspelled routing path field: - match: { hits.total.value: 6 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key._tsid: "_kNKbD0oaZ29OmgpnMwz_lb52slXKOVHbNg" } + - match: { aggregations.tsids.buckets.0.key._tsid: "_pW31QsQVpe7uQk6zUIFfFcpASZ8Y_6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAoaZ29_pW31QsQ6X1JYjFYp22EdW9Qe7Fh3Q" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "_kNKbD2rnf7qOrrF1oUBkaXNQ_lHN_ixdkU" } + - match: { aggregations.tsids.buckets.1.key._tsid: "_pW31QsQVpe7uQk6zUIFfFcpASZ8Y_6Vt9ULEAAAAAAAAAAAAAAAAAAAAACrnf7q_pW31QsQTp_Y-T2FMaY2THS8Sv5rUQ" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "_kNKbD3P1l1UOghnGo67t0KITB0qgux2pmo" } + - match: { aggregations.tsids.buckets.2.key._tsid: "_pW31QsQVpe7uQk6zUIFfFcpASZ8Y_6Vt9ULEAAAAAAAAAAAAAAAAAAAAADP1l1U_pW31QsQvSkqmNP15820DPPjhXpE7w" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "_kNKbD3gKelzOkE02kX5dIA6LsI0s8xNPUA" } + - match: { aggregations.tsids.buckets.3.key._tsid: "_pW31QsQVpe7uQk6zUIFfFcpASZ8Y_6Vt9ULEAAAAAAAAAAAAAAAAAAAAADgKelz_pW31QsQ59jWtAPPEZOFAL6Fp-tkHA" } - match: { aggregations.tsids.buckets.3.doc_count: 1 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} @@ -529,10 +529,10 @@ long dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "4Yt2mD1DgKYdOlaZAWQ973outjqByH6Leukt8Syv6_LtDjuE"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "_pW31QsQXNiiZwxkczx2lrb8CBwni_6Vt9ULEAAAAAAAAAAAAAAAAAAAAABDgKYd73outv6Vt9ULEBnwPdwtIAd7vn-6M1jNT1s"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "4Yt2mD2tm41vOlaZAWQ973outjrbyf07xExY7uwo0Fb0U9i9"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "_pW31QsQXNiiZwxkczx2lrb8CBwni_6Vt9ULEAAAAAAAAAAAAAAAAAAAAACtm41v73outv6Vt9ULELmJNwhM2k9uEB39QGAP8PY"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -604,10 +604,10 @@ ip dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: { _tsid: "4h8Hlj3Yzb-vOlaZAWQ973outjpBT4iJTy1L0m3hsXJt8DKM"}} + - match: {aggregations.tsids.buckets.0.key: { _tsid: "_pW31QsQWlq2NkhwVgBD7x8-uwLcHf6Vt9ULEAAAAAAAAAAAAAAAAAAAAADYzb-v73outv6Vt9ULEBko4DlimDxhOKUBMl0caik"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: { _tsid: "4h8Hlj3aaOFKOlaZAWQ973outjq8M4UoPsTYTRpxgCKGd65O"}} + - match: {aggregations.tsids.buckets.1.key: { _tsid: "_pW31QsQWlq2NkhwVgBD7x8-uwLcHf6Vt9ULEAAAAAAAAAAAAAAAAAAAAADaaOFK73outv6Vt9ULEJocP_SSPwvd1Uk-BTEqof0"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index f3944cb4c5c95..015b7fbfd3d85 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -123,7 +123,7 @@ shrink: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}]} --- clone: @@ -147,4 +147,4 @@ clone: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}]} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java index 0eb9b7788cd83..1759beb2cbedf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java @@ -94,7 +94,7 @@ public void testTotalNumberOfDimensionFieldsLimit() { } public void testTotalNumberOfDimensionFieldsDefaultLimit() { - int dimensionFieldLimit = 21; + int dimensionFieldLimit = 9999; final Exception ex = expectThrows(IllegalArgumentException.class, () -> createTimeSeriesIndex(mapping -> { mapping.startObject("routing_field").field("type", "keyword").field("time_series_dimension", true).endObject(); for (int i = 0; i < dimensionFieldLimit; i++) { @@ -175,6 +175,7 @@ private void createTimeSeriesIndex( Settings.Builder settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 11_000) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPaths.get()) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2000-01-08T23:40:53.384Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z"); diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 05afc14e0f0cd..118433f6e18ef 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -17,7 +17,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.DocumentDimensions; +import org.elasticsearch.index.mapper.DocumentFields; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperService; @@ -103,8 +103,8 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public DocumentDimensions buildDocumentDimensions(IndexSettings settings) { - return new DocumentDimensions.OnlySingleValueAllowed(); + public DocumentFields buildDocumentDimensions(IndexSettings settings) { + return new DocumentFields.OnlySingleValueAllowed(); } @Override @@ -196,7 +196,7 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public DocumentDimensions buildDocumentDimensions(IndexSettings settings) { + public DocumentFields buildDocumentDimensions(IndexSettings settings) { IndexRouting.ExtractFromSource routing = (IndexRouting.ExtractFromSource) settings.getIndexRouting(); return new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routing.builder()); } @@ -325,7 +325,7 @@ public String getName() { /** * How {@code time_series_dimension} fields are handled by indices in this mode. */ - public abstract DocumentDimensions buildDocumentDimensions(IndexSettings settings); + public abstract DocumentFields buildDocumentDimensions(IndexSettings settings); /** * @return Whether timestamps should be validated for being withing the time range of an index. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentFields.java similarity index 63% rename from server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java rename to server/src/main/java/org/elasticsearch/index/mapper/DocumentFields.java index fafa7f7a9cb12..434a74b80e8b7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentFields.java @@ -18,12 +18,12 @@ /** * Collects dimensions from documents. */ -public interface DocumentDimensions { +public interface DocumentFields { /** * Build an index's DocumentDimensions using its settings */ - static DocumentDimensions fromIndexSettings(IndexSettings indexSettings) { + static DocumentFields fromIndexSettings(IndexSettings indexSettings) { return indexSettings.getMode().buildDocumentDimensions(indexSettings); } @@ -32,47 +32,54 @@ static DocumentDimensions fromIndexSettings(IndexSettings indexSettings) { * value is already computed in some cases when we want to collect * dimensions, so we can save re-computing the UTF-8 encoding. */ - void addString(String fieldName, BytesRef utf8Value); + void addKeywordDimension(String fieldName, BytesRef utf8Value); - default void addString(String fieldName, String value) { - addString(fieldName, new BytesRef(value)); + default void addKeywordDimension(String fieldName, String value) { + addKeywordDimension(fieldName, new BytesRef(value)); } - void addIp(String fieldName, InetAddress value); + void addIpDimension(String fieldName, InetAddress value); - void addLong(String fieldName, long value); + void addLongDimension(String fieldName, long value); - void addUnsignedLong(String fieldName, long value); + void addUnsignedLongDimension(String fieldName, long value); + + void addMetric(String fieldName); /** * Makes sure that each dimension only appears on time. */ - class OnlySingleValueAllowed implements DocumentDimensions { + class OnlySingleValueAllowed implements DocumentFields { private final Set names = new HashSet<>(); @Override - public void addString(String fieldName, BytesRef value) { + public void addKeywordDimension(String fieldName, BytesRef value) { add(fieldName); } // Override to skip the UTF-8 conversion that happens in the default implementation @Override - public void addString(String fieldName, String value) { + public void addKeywordDimension(String fieldName, String value) { + add(fieldName); + } + + @Override + public void addIpDimension(String fieldName, InetAddress value) { add(fieldName); } @Override - public void addIp(String fieldName, InetAddress value) { + public void addLongDimension(String fieldName, long value) { add(fieldName); } @Override - public void addLong(String fieldName, long value) { + public void addUnsignedLongDimension(String fieldName, long value) { add(fieldName); } @Override - public void addUnsignedLong(String fieldName, long value) { + public void addMetric(String fieldName) { add(fieldName); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 933f97d46dc08..cc0332376754b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -88,7 +88,7 @@ protected void addDoc(LuceneDocument doc) { private final Set newFieldsSeen; private final Map dynamicObjectMappers; private final List dynamicRuntimeFields; - private final DocumentDimensions dimensions; + private final DocumentFields documentFields; private final ObjectMapper parent; private final ObjectMapper.Dynamic dynamic; private String id; @@ -107,7 +107,7 @@ private DocumentParserContext( String id, Field version, SeqNoFieldMapper.SequenceIDFields seqID, - DocumentDimensions dimensions, + DocumentFields documentFields, ObjectMapper parent, ObjectMapper.Dynamic dynamic ) { @@ -122,7 +122,7 @@ private DocumentParserContext( this.id = id; this.version = version; this.seqID = seqID; - this.dimensions = dimensions; + this.documentFields = documentFields; this.parent = parent; this.dynamic = dynamic; } @@ -140,7 +140,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, in.id, in.version, in.seqID, - in.dimensions, + in.documentFields, parent, dynamic ); @@ -165,7 +165,7 @@ protected DocumentParserContext( null, null, null, - DocumentDimensions.fromIndexSettings(mappingParserContext.getIndexSettings()), + DocumentFields.fromIndexSettings(mappingParserContext.getIndexSettings()), parent, dynamic ); @@ -481,8 +481,8 @@ public XContentParser parser() { /** * The collection of dimensions for this document. */ - public DocumentDimensions getDimensions() { - return dimensions; + public DocumentFields getDocumentFields() { + return documentFields; } public abstract ContentPath path(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 259f51027ce73..4e3d392e1188f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -546,7 +546,7 @@ private static InetAddress value(XContentParser parser, InetAddress nullValue) t private void indexValue(DocumentParserContext context, InetAddress address) { if (dimension) { - context.getDimensions().addIp(fieldType().name(), address); + context.getDocumentFields().addIpDimension(fieldType().name(), address); } if (indexed) { Field field = new InetAddressPoint(fieldType().name(), address); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 31ca329d03375..15b6a4e68dd24 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -887,7 +887,7 @@ private void indexValue(DocumentParserContext context, String value) { final BytesRef binaryValue = new BytesRef(value); if (fieldType().isDimension()) { - context.getDimensions().addString(fieldType().name(), binaryValue); + context.getDocumentFields().addKeywordDimension(fieldType().name(), binaryValue); } // If the UTF8 encoding of the field value is bigger than the max length 32766, Lucene fill fail the indexing request and, to diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 7590fd36d849a..c7897655f8844 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1793,7 +1793,9 @@ public Number value(XContentParser parser) throws IllegalArgumentException, IOEx */ public void indexValue(DocumentParserContext context, Number numericValue) { if (dimension && numericValue != null) { - context.getDimensions().addLong(fieldType().name(), numericValue.longValue()); + context.getDocumentFields().addLongDimension(fieldType().name(), numericValue.longValue()); + } else if (fieldType().getMetricType() != null) { + context.getDocumentFields().addMetric(fieldType().name()); } fieldType().type.addFields(context.doc(), fieldType().name(), numericValue, indexed, hasDocValues, stored); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index e92ff8fd601ae..412a0193025c8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -10,7 +10,6 @@ import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.search.Query; -import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; import org.elasticsearch.cluster.routing.IndexRouting; @@ -42,6 +41,7 @@ import java.util.Comparator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -56,17 +56,7 @@ public class TimeSeriesIdFieldMapper extends MetadataFieldMapper { public static final TimeSeriesIdFieldType FIELD_TYPE = new TimeSeriesIdFieldType(); public static final TimeSeriesIdFieldMapper INSTANCE = new TimeSeriesIdFieldMapper(); - /** - * The maximum length of the tsid. The value itself comes from a range check in - * Lucene's writer for utf-8 doc values. - */ - private static final int LIMIT = ByteBlockPool.BYTE_BLOCK_SIZE - 2; - /** - * The maximum length of any single dimension. We picked this so that we could - * comfortable fit 16 dimensions inside {@link #LIMIT}. This should be quite - * comfortable given that dimensions are typically going to be less than a - * hundred bytes each, but we're being paranoid here. - */ + // NOTE: used by {@link TimeSeriesIdFieldMapper#decodeTsid(StreamInput)} )}. Remove both if using _tsid hashing public static final int TSID_HASH_SENTINEL = 0xBAADCAFE; @Override @@ -144,27 +134,10 @@ private TimeSeriesIdFieldMapper() { public void postParse(DocumentParserContext context) throws IOException { assert fieldType().isIndexed() == false; - final TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDimensions(); - final BytesReference timeSeriesId = timeSeriesIdBuilder.build(); - context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesIdBuilder.similarityHash(timeSeriesId).toBytesRef())); - TsidExtractingIdFieldMapper.createField(context, timeSeriesIdBuilder.routingBuilder, timeSeriesId.toBytesRef()); - } - - // TODO: remove if using {@link TimeSeriesIdBuilder#similarityHash(BytesReference)} - public static BytesReference hash128(final BytesReference timeSeriesId) throws IOException { - try (BytesStreamOutput out = new BytesStreamOutput()) { - final byte[] buffer = new byte[16]; - out.writeVInt(TSID_HASH_SENTINEL); - final BytesRef tsid = timeSeriesId.toBytesRef(); - final MurmurHash3.Hash128 hash = new MurmurHash3.Hash128(); - MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, 0, hash); - ByteUtils.writeLongLE(hash.h1, buffer, 0); - ByteUtils.writeLongLE(hash.h2, buffer, 8); - // TODO: maybe remove Base64 encoding and do it in {@link TimeSeriesIdFieldMapper#decodeTsid(StreamInput)} )} - final BytesRef encoded = new BytesRef(Base64.getUrlEncoder().withoutPadding().encodeToString(buffer)); - out.writeBytesRef(encoded); - return out.bytes(); - } + final TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDocumentFields(); + final BytesRef timeSeriesId = timeSeriesIdBuilder.build().toBytesRef(); + context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesIdBuilder.similarityHash().toBytesRef())); + TsidExtractingIdFieldMapper.createField(context, timeSeriesIdBuilder.routingBuilder, timeSeriesId); } @Override @@ -211,18 +184,19 @@ public static Map decodeTsid(StreamInput in) { } } - public static class TimeSeriesIdBuilder implements DocumentDimensions { + public static class TimeSeriesIdBuilder implements DocumentFields { public static final int MAX_DIMENSIONS = 512; - private record DimensionDataHolder(BytesReference fieldName, int fieldNameHash, BytesReference value, int valueHash) {} + private record DimensionDataHolder(String name, BytesReference value) {} /** * A sorted set of the serialized values of dimension fields that will be used * for generating the _tsid field. The map will be used by {@link TimeSeriesIdFieldMapper} * to build the _tsid field for the document. */ - private final SortedSet dimensions = new TreeSet<>(Comparator.comparing(o -> o.fieldName)); + private final SortedSet dimensions = new TreeSet<>(Comparator.comparing(o -> o.name)); + private final Set metrics = new TreeSet<>(); /** * Builds the routing. Used for building {@code _id}. If null then skipped. */ @@ -241,7 +215,7 @@ public BytesReference build() throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(dimensions.size()); for (DimensionDataHolder entry : dimensions) { - out.writeBytesRef(entry.fieldName.toBytesRef()); + out.writeBytesRef(new BytesRef(entry.name)); entry.value.writeTo(out); } return out.bytes(); @@ -251,40 +225,99 @@ public BytesReference build() throws IOException { /** * Here we build the hash of the tsid using a similarity function so that we have a result * with the following pattern: - * ${similarityHash(_tsid)}-${hash(_tsid)}. + * + * hash128(catenate(dimension field names)) + + * hash128(catenate(metric field names)) + + * foreach(dimension field value, limit = MAX_DIMENSIONS) { hash32(dimension field value) } + + * hash128(catenate(dimension field values)) + * * The idea is to be able to place 'similar' time series close to each other. Two time series - * are considered 'similar' if they share the same values for a subset of the dimensions (sorted - * names/values). + * are considered 'similar' if they share the same dimensions (names and values). */ - public BytesReference similarityHash(final BytesReference timeSeriesId) throws IOException { - int bufferIndex = 0; - // max 512 entries of (hash32(fieldName) + '=' + hash32(fieldValue) + ":") plus the has128(timeSeriesId) - final byte[] buffer = new byte[MAX_DIMENSIONS * 10 + 16]; - for (final DimensionDataHolder dimensionDataHolder : dimensions) { - if (bufferIndex >= MAX_DIMENSIONS) break; - ByteUtils.writeIntLE(dimensionDataHolder.fieldNameHash, buffer, bufferIndex); - ByteUtils.writeIntLE('=', buffer, bufferIndex + 4); - ByteUtils.writeIntLE(dimensionDataHolder.valueHash, buffer, bufferIndex + 5); - ByteUtils.writeIntLE(':', buffer, bufferIndex + 9); - bufferIndex += 10; + public BytesReference similarityHash() throws IOException { + // NOTE: hash all dimension field names + int dimensionValuesLength = 0; + final StringBuilder sb = new StringBuilder(); + for (final DimensionDataHolder dimension : dimensions) { + final BytesRef bytesRef = dimension.value.toBytesRef(); + dimensionValuesLength += (bytesRef.length - bytesRef.offset); + sb.append(dimension.name); + } + final BytesReference dimensionNamesHash = hash128(new BytesRef(sb.toString())); + + // NOTE: hash all metric field names + sb.setLength(0); + for (final String metric : metrics) { + sb.append(metric); + } + final BytesReference metricFieldsHash = hash128(new BytesRef(sb.toString())); + + // NOTE: catenate all dimension value hashes up to names certain number of dimensions + int numberOfDimensions = Math.min(MAX_DIMENSIONS, dimensions.size()); + int buf1Index = 0; + final byte[] buf1 = new byte[4 * numberOfDimensions]; + for (final DimensionDataHolder dimension : dimensions) { + if (buf1Index >= buf1.length) break; + final BytesRef dimensionValueBytesRef = dimension.value().toBytesRef(); + ByteUtils.writeIntLE(StringHelper.murmurhash3_x86_32(dimensionValueBytesRef, 0), buf1, buf1Index); + buf1Index += 4; + } + final BytesRef valueByValue = new BytesRef(buf1, 0, buf1.length); + + // NOTE: hash all dimension field allValues + int buf2Index = 0; + final byte[] buf2 = new byte[dimensionValuesLength]; + for (final DimensionDataHolder dimension : dimensions) { + final BytesRef dimensionValueBytesRef = dimension.value.toBytesRef(); + System.arraycopy( + dimensionValueBytesRef.bytes, + dimensionValueBytesRef.offset, + buf2, + buf2Index, + dimensionValueBytesRef.length - dimensionValueBytesRef.offset + ); + buf2Index += (dimensionValueBytesRef.length - dimensionValueBytesRef.offset); } - final BytesRef timeSeriesIdBytesRef = timeSeriesId.toBytesRef(); - final MurmurHash3.Hash128 tsidFullHash = new MurmurHash3.Hash128(); - MurmurHash3.hash128(timeSeriesIdBytesRef.bytes, timeSeriesIdBytesRef.offset, timeSeriesIdBytesRef.length, 0, tsidFullHash); - ByteUtils.writeLongLE(tsidFullHash.h1, buffer, bufferIndex); - bufferIndex += 8; - ByteUtils.writeLongLE(tsidFullHash.h2, buffer, bufferIndex); - bufferIndex += 8; + final BytesReference dimensionValuesHash = hash128(new BytesRef(buf2, 0, buf2.length)); + + final BytesRef names = dimensionNamesHash.toBytesRef(); + final BytesRef metrics = metricFieldsHash.toBytesRef(); + final BytesRef allValues = dimensionValuesHash.toBytesRef(); + int tsidLength = names.length + metrics.length + valueByValue.length + allValues.length; + byte[] tsid = new byte[tsidLength]; + int tsidIndex = 0; + System.arraycopy(names.bytes, names.offset, tsid, tsidIndex, names.length); + tsidIndex += names.length; + System.arraycopy(metrics.bytes, metrics.offset, tsid, tsidIndex, metrics.length); + tsidIndex += metrics.length; + System.arraycopy(valueByValue.bytes, valueByValue.offset, tsid, tsidIndex, valueByValue.length); + tsidIndex += valueByValue.length; + System.arraycopy(allValues.bytes, allValues.offset, tsid, tsidIndex, allValues.length); + tsidIndex += allValues.length; + assert tsidIndex == tsidLength; try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(TSID_HASH_SENTINEL); - final BytesRef hash = new BytesRef(Arrays.copyOfRange(buffer, 0, bufferIndex)); - out.writeBytesRef(hash); + out.writeBytesRef(new BytesRef(tsid, 0, tsidLength)); + return out.bytes(); + } + } + + private BytesReference hash128(final BytesRef value) throws IOException { + byte[] buffer = new byte[16]; + final MurmurHash3.Hash128 hash128 = new MurmurHash3.Hash128(); + MurmurHash3.hash128(value.bytes, value.offset, value.length, 0, hash128); + ByteUtils.writeLongLE(hash128.h1, buffer, 0); + ByteUtils.writeLongLE(hash128.h2, buffer, 8); + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(TSID_HASH_SENTINEL); + final BytesRef hashBytesRef = new BytesRef(Arrays.copyOfRange(buffer, 0, buffer.length)); + out.writeBytesRef(hashBytesRef); return out.bytes(); } } @Override - public void addString(String fieldName, BytesRef utf8Value) { + public void addKeywordDimension(String fieldName, BytesRef utf8Value) { try (BytesStreamOutput out = new BytesStreamOutput()) { out.write((byte) 's'); /* @@ -304,12 +337,12 @@ public void addString(String fieldName, BytesRef utf8Value) { } @Override - public void addIp(String fieldName, InetAddress value) { - addString(fieldName, NetworkAddress.format(value)); + public void addIpDimension(String fieldName, InetAddress value) { + addKeywordDimension(fieldName, NetworkAddress.format(value)); } @Override - public void addLong(String fieldName, long value) { + public void addLongDimension(String fieldName, long value) { try (BytesStreamOutput out = new BytesStreamOutput()) { out.write((byte) 'l'); out.writeLong(value); @@ -320,7 +353,7 @@ public void addLong(String fieldName, long value) { } @Override - public void addUnsignedLong(String fieldName, long value) { + public void addUnsignedLongDimension(String fieldName, long value) { try (BytesStreamOutput out = new BytesStreamOutput()) { Object ul = DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value); if (ul instanceof Long l) { @@ -336,14 +369,13 @@ public void addUnsignedLong(String fieldName, long value) { } } + @Override + public void addMetric(String fieldName) { + metrics.add(fieldName); + } + private void add(String fieldName, BytesReference encoded) throws IOException { - final BytesArray fieldNameBytesRef = new BytesArray(fieldName); - final DimensionDataHolder dimension = new DimensionDataHolder( - fieldNameBytesRef, - StringHelper.murmurhash3_x86_32(fieldNameBytesRef.toBytesRef(), 0), - encoded, - StringHelper.murmurhash3_x86_32(encoded.toBytesRef(), 0) - ); + final DimensionDataHolder dimension = new DimensionDataHolder(fieldName, encoded); if (dimensions.contains(dimension)) { throw new IllegalArgumentException("Dimension field [" + fieldName + "] cannot be a multi-valued field."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java index f09c6f8c036c8..389a223ddc182 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java @@ -173,7 +173,7 @@ private void addField(DocumentParserContext context, ContentPath path, String cu final String keyedFieldName = FlattenedFieldParser.extractKey(bytesKeyedValue).utf8ToString(); if (fieldType.isDimension() && fieldType.dimensions().contains(keyedFieldName)) { final BytesRef keyedFieldValue = FlattenedFieldParser.extractValue(bytesKeyedValue); - context.getDimensions().addString(rootFieldName + "." + keyedFieldName, keyedFieldValue); + context.getDocumentFields().addKeywordDimension(rootFieldName + "." + keyedFieldName, keyedFieldValue); } } } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 651bcfbef045d..d55709bd372f4 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -710,14 +710,14 @@ public BytesRef parseBytesRef(Object value) { Object v = entry.getValue(); if (v instanceof String s) { - builder.addString(f, s); + builder.addKeywordDimension(f, s); } else if (v instanceof Long l) { - builder.addLong(f, l); + builder.addLongDimension(f, l); } else if (v instanceof Integer i) { - builder.addLong(f, i.longValue()); + builder.addLongDimension(f, i.longValue()); } else if (v instanceof BigInteger ul) { long ll = UNSIGNED_LONG_SHIFTED.parseLong(ul.toString(), false, () -> 0L); - builder.addUnsignedLong(f, ll); + builder.addUnsignedLongDimension(f, ll); } else { throw new IllegalArgumentException("Unexpected value in tsid object [" + v + "]"); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index 8c7df193f8d84..f30d4cfb119a3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -229,10 +229,10 @@ private static void indexDoc(IndexRouting.ExtractFromSource routing, RandomIndex fields.add(new LongPoint(DataStreamTimestampFieldMapper.DEFAULT_PATH, doc.timestamp)); for (Dimension dimension : doc.dimensions) { if (dimension.value instanceof Number n) { - builder.addLong(dimension.field, n.longValue()); + builder.addLongDimension(dimension.field, n.longValue()); fields.add(new SortedNumericDocValuesField(dimension.field, ((Number) dimension.value).longValue())); } else { - builder.addString(dimension.field, dimension.value.toString()); + builder.addKeywordDimension(dimension.field, dimension.value.toString()); fields.add(new SortedSetDocValuesField(dimension.field, new BytesRef(dimension.value.toString()))); } } @@ -246,9 +246,9 @@ private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routingBuilder); for (Dimension dimension : doc.dimensions) { if (dimension.value instanceof Number n) { - timeSeriesIdBuilder.addLong(dimension.field, n.longValue()); + timeSeriesIdBuilder.addLongDimension(dimension.field, n.longValue()); } else { - timeSeriesIdBuilder.addString(dimension.field, dimension.value.toString()); + timeSeriesIdBuilder.addKeywordDimension(dimension.field, dimension.value.toString()); } } return TsidExtractingIdFieldMapper.createId( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 82d4d8e2acbec..ffaf797f5d84d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -78,7 +78,10 @@ public void testEnabledInTimeSeriesMode() throws Exception { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100).field("c", 500)); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "smklPD3uDjjxOgN-3pU9PwT2Ojp-syegKjheHg5h5ERLJH6q") + matchesMap().entry( + "_tsid", + "_pW31QsQLhvtFuoRi5Ot1FKbAade5v6Vt9ULEAAAAAAAAAAAAAAAAAAAAADuDjjxPwT2Ov6Vt9ULEMF14BMjwbnf88_kkYIqqxM" + ) ); } @@ -125,7 +128,10 @@ public void testStrings() throws IOException { ); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "smklPD2v9houOmeyDcA9tA-WtDo47R9yulyYUoudsXFaYRtH") + matchesMap().entry( + "_tsid", + "_pW31QsQ-uh8SqJqlbcmNjsxQifzOf6Vt9ULEAAAAAAAAAAAAAAAAAAAAACv9houtA-WtP6Vt9ULEP9c3WHBQBL_APxECsSTB9g" + ) ); } @@ -141,7 +147,13 @@ public void testUnicodeKeys() throws IOException { Map tsid = TimeSeriesIdFieldMapper.decodeTsid( new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes) ); - assertMap(tsid, matchesMap().entry("_tsid", "2vZdwj2ejAXjOviZF9Y9DKlQ6jqs_5V1RcDg-M6kBhZ5cO40")); + assertMap( + tsid, + matchesMap().entry( + "_tsid", + "_pW31QsQJp7d1TeAbnqXtvYiILWo7v6Vt9ULEAAAAAAAAAAAAAAAAAAAAACejAXjDKlQ6v6Vt9ULEAYFWdDsVn7JHKYuKWckZQA" + ) + ); } public void testKeywordTooLong() throws IOException { @@ -152,7 +164,7 @@ public void testKeywordTooLong() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "smklPD29ZDUoOsglT7sfXPKk_XyMzgZv1OM") + matchesMap().entry("_tsid", "_pW31QsQiXhZ9mVVVYVaiQ5RSDq15v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAC9ZDUo_pW31QsQhD-XZN6wU-KM0X_aiyysUQ") ); } @@ -165,7 +177,7 @@ public void testKeywordTooLongUtf8() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "smklPD1M43f6OplCG-jkf0ftosS7HTvPX9Q") + matchesMap().entry("_tsid", "_pW31QsQiXhZ9mVVVYVaiQ5RSDq15v6Vt9ULEAAAAAAAAAAAAAAAAAAAAABM43f6_pW31QsQUqNsQMp1y6DdXAgTdWKAHg") ); } @@ -206,7 +218,10 @@ public void testLong() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "smklPD2tm41vOuD2FrQ9SrpJyzpnsg3APd1wa2A6wmulw_SDY7Ceqg2_cpDqoQ") + matchesMap().entry( + "_tsid", + "_pW31QsQy2oH44MiIyaUP_cSq588Qf6Vt9ULEAAAAAAAAAAAAAAAAAAAAACtm41vSrpJy91wa2D-lbfVCxBM5cSB4LNLdZ9rW4UIsHrm" + ) ); } @@ -260,7 +275,10 @@ public void testInteger() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "smklPD2tm41vOuD2FrQ9SrpJyzpnsg3APdXyEVA6iVca07m_g8ACVY2SDNSrWw") + matchesMap().entry( + "_tsid", + "_pW31QsQy2oH44MiIyaUP_cSq588Qf6Vt9ULEAAAAAAAAAAAAAAAAAAAAACtm41vSrpJy9XyEVD-lbfVCxBvG-5-7IUnwc1jT9EFsPcy" + ) ); } @@ -318,7 +336,10 @@ public void testShort() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "smklPD2tm41vOuD2FrQ9SrpJyzpnsg3APSYAWY86NJtZIt97lRcRUmcsL5Wx2Q") + matchesMap().entry( + "_tsid", + "_pW31QsQy2oH44MiIyaUP_cSq588Qf6Vt9ULEAAAAAAAAAAAAAAAAAAAAACtm41vSrpJyyYAWY_-lbfVCxBnXXrTqLOmwy6CMvTbNEBr" + ) ); } @@ -376,7 +397,10 @@ public void testByte() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "smklPD2tm41vOuD2FrQ9SrpJyzpnsg3APSgrqoc6Jd0t9U26YHnblyf7j8D6Nw") + matchesMap().entry( + "_tsid", + "_pW31QsQy2oH44MiIyaUP_cSq588Qf6Vt9ULEAAAAAAAAAAAAAAAAAAAAACtm41vSrpJyygrqof-lbfVCxAepXyuslV4c5JCMrunm5oE" + ) ); } @@ -434,7 +458,10 @@ public void testIp() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "smklPD0LrOlWOuD2FrQ9SrpJyzpnsg3APbPzTuw6XwWVVVRBrzAohq4krNO_Vg") + matchesMap().entry( + "_tsid", + "_pW31QsQy2oH44MiIyaUP_cSq588Qf6Vt9ULEAAAAAAAAAAAAAAAAAAAAAALrOlWSrpJy7PzTuz-lbfVCxBolHEPXKdw6u7TkGuW-CJ7" + ) ); } @@ -472,7 +499,7 @@ public void testVeryLarge() throws IOException { TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), matchesMap().entry( "_tsid", - "A37elT2v9houOt_AxbA9WKlBkzpDxtikPVipQZM63evtJj1YqUGTOmcGd009WKlBkzoL00iOPVipQZM6HcyYDT1YqUGTOmP60TU9WKlBkzomGtdvPVipQZM6nzBSpz1YqUGTOhCUdPc9WKlBkzrt94dlPVipQZM6tpOXzz1YqUGTOh3AKiU9WKlBkzqzeblUPVipQZM64cqTnD1YqUGTOo7laSA9WKlBkzrsUq91PVipQZM6werJIj1YqUGTOv35ZJw9WKlBkzqVPxV2PVipQZM6EXQ--T1YqUGTOrQj-k49WKlBkzpvCEIlPVipQZM6aBOtmD1YqUGTOuPkbfc9WKlBkzpbjZwZPVipQZM69D3Ilz1YqUGTOuxzV_s9WKlBkzrSfQp1PVipQZM6_cSVAD1YqUGTOnC5_2s9WKlBkzrN0KotPVipQZM6YDdNnj1YqUGTOgHxRIM9WKlBkzofqtnuPVipQZM6Ol1y0z1YqUGTOocA5Ws9WKlBkzo_lx9_PVipQZM6vYmcTj1YqUGTOraOdYA9WKlBkzq4XTKzPVipQZM6IAcd1j1YqUGTOmJz9-09WKlBkzo3MbjRPVipQZM63bx19j1YqUGTOuBxVpk9WKlBkzoGxKMZPVipQZM6QrzUnD1YqUGTOvRyFPE9WKlBkzo6syljPVipQZM6tgdVsT1YqUGTOiNB-T6bYXc3nUOdLzDvWN0" + "_pW31QsQVuMQB6AxmJbsu4Fzii310f6Vt9ULEAAAAAAAAAAAAAAAAAAAAACv9houWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk_6Vt9ULEOAOK-X4tmuBNmIlX0uEX_s" ) ); } @@ -571,6 +598,47 @@ public void testDifferentValues() throws IOException { assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); } + public void testSameMetricNamesDifferentValues() throws IOException { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("m1").field("type", "double").field("time_series_metric", "gauge").endObject(); + b.startObject("m2").field("type", "integer").field("time_series_metric", "counter").endObject(); + })); + + ParsedDocument doc1 = parseDocument( + docMapper, + d -> d.field("a", "value") + .field("b", 10) + .field("m1", randomDoubleBetween(100, 200, true)) + .field("m2", randomIntBetween(100, 200)) + ); + ParsedDocument doc2 = parseDocument( + docMapper, + d -> d.field("a", "value").field("b", 10).field("m1", randomDoubleBetween(10, 20, true)).field("m2", randomIntBetween(10, 20)) + ); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } + + public void testDifferentMetricNamesSameValues() throws IOException { + DocumentMapper docMapper1 = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("m1").field("type", "double").field("time_series_metric", "gauge").endObject(); + })); + + DocumentMapper docMapper2 = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("m2").field("type", "double").field("time_series_metric", "gauge").endObject(); + })); + + double metricValue = randomDoubleBetween(10, 20, true); + ParsedDocument doc1 = parseDocument(docMapper1, d -> d.field("a", "value").field("b", 10).field("m1", metricValue)); + ParsedDocument doc2 = parseDocument(docMapper2, d -> d.field("a", "value").field("b", 10).field("m2", metricValue)); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } + /** * Two documents with the same *values* but different dimension keys will generate * different {@code _tsid}s. diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index def373a6cc861..ad3ba6107c8a8 100644 --- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -374,9 +374,9 @@ public void testParseZone() { public void testParseTsid() throws IOException { TimeSeriesIdBuilder timeSeriesIdBuilder = new TimeSeriesIdBuilder(null); - timeSeriesIdBuilder.addString("string", randomAlphaOfLength(10)); - timeSeriesIdBuilder.addLong("long", randomLong()); - timeSeriesIdBuilder.addUnsignedLong("ulong", randomLong()); + timeSeriesIdBuilder.addKeywordDimension("string", randomAlphaOfLength(10)); + timeSeriesIdBuilder.addLongDimension("long", randomLong()); + timeSeriesIdBuilder.addUnsignedLongDimension("ulong", randomLong()); BytesRef tsidBytes = timeSeriesIdBuilder.build().toBytesRef(); Object tsidFormat = DocValueFormat.TIME_SERIES_ID.format(tsidBytes); BytesRef tsidParse = DocValueFormat.TIME_SERIES_ID.parseBytesRef(tsidFormat); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index 32a337f0b329b..1151ddcb57507 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -121,7 +121,7 @@ private List docs(long startTimestamp, String dim, long... values) thr private static BytesReference tsid(String dim) throws IOException { TimeSeriesIdFieldMapper.TimeSeriesIdBuilder idBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); - idBuilder.addString("dim", dim); + idBuilder.addKeywordDimension("dim", dim); return idBuilder.build(); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 04554b577b481..83477b53936f4 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -610,7 +610,9 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } if (dimension && numericValue != null) { - context.getDimensions().addUnsignedLong(fieldType().name(), numericValue); + context.getDocumentFields().addUnsignedLongDimension(fieldType().name(), numericValue); + } else if (fieldType().getMetricType() != null) { + context.getDocumentFields().addMetric(fieldType().name()); } List fields = new ArrayList<>(); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index 2d6bbaf8ccd97..64b83a29584f6 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -799,7 +799,7 @@ private void assertGeoLine_TSDB( ArrayList timestamps = testData.timestampsForGroup(g); for (int i = 0; i < points.size(); i++) { final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); - builder.addString("group_id", testData.groups[g]); + builder.addKeywordDimension("group_id", testData.groups[g]); ArrayList fields = new ArrayList<>( Arrays.asList( new SortedDocValuesField("group_id", new BytesRef(testData.groups[g])), From 5e224c63ff3d23772c52f02b5bfbd5375d330e30 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 19 Oct 2023 11:27:38 +0200 Subject: [PATCH 018/125] fix: incorrect hash128 --- .../test/aggregations/time_series.yml | 10 ++-- .../test/data_stream/150_tsdb.yml | 6 +- .../test/tsdb/25_id_generation.yml | 6 +- .../rest-api-spec/test/tsdb/30_snapshot.yml | 2 +- .../rest-api-spec/test/tsdb/40_search.yml | 14 ++--- .../rest-api-spec/test/tsdb/50_alias.yml | 10 ++-- .../test/tsdb/60_add_dimensions.yml | 10 ++-- .../test/tsdb/70_dimension_types.yml | 44 +++++++------- .../test/tsdb/80_index_resize.yml | 4 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 60 +++++++------------ .../mapper/TimeSeriesIdFieldMapperTests.java | 49 ++++----------- 11 files changed, 87 insertions(+), 128 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 67f648f7e4b74..f5a8d99b55b2c 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -76,7 +76,7 @@ setup: - match: { hits.total.value: 1 } - length: { aggregations: 1 } - - match: { aggregations.ts.buckets.0.key: { _tsid: "RGqc4j2v9houOhyLbojV0RSqi9yhwd0OzmU" } } + - match: { aggregations.ts.buckets.0.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACv9houG2TKanX0IDDfq7mPr0SxJA" } } - match: { aggregations.ts.buckets.0.doc_count: 1 } --- @@ -126,7 +126,7 @@ setup: size: 1 - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: { _tsid: "RGqc4j2Kf_6WOnwsPSufD8_yG6Y_INhnbbI" } } + - match: { aggregations.ts.buckets.0.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACKf_6WEKY54y0GuGgKb5oO6t-5YQ" } } - do: search: @@ -144,9 +144,9 @@ setup: size: 3 - length: { aggregations.ts.buckets: 3 } - - match: { aggregations.ts.buckets.0.key: { _tsid: "RGqc4j2Kf_6WOnwsPSufD8_yG6Y_INhnbbI" } } - - match: { aggregations.ts.buckets.1.key: { _tsid: "RGqc4j2v9houOhyLbojV0RSqi9yhwd0OzmU" } } - - match: { aggregations.ts.buckets.2.key: { _tsid: "RGqc4j25AZB2OoOAyhyNewG5H884opnvlxE" } } + - match: { aggregations.ts.buckets.0.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACKf_6WEKY54y0GuGgKb5oO6t-5YQ" } } + - match: { aggregations.ts.buckets.1.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACv9houG2TKanX0IDDfq7mPr0SxJA" } } + - match: { aggregations.ts.buckets.2.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAAC5AZB2QuYbdK0MxkFW7lg9z6q4rA" } } --- "Score test filter some": diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index cac3cb649804e..1ab73edce3add 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -143,7 +143,7 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39" }]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g" }]} --- "aggregate the tsid": @@ -164,9 +164,9 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "q2RfQj05ihHDOlaZAWQ9--qoyDozlLccrbi5EMFdqCeotpKt"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "q2RfQj112oDhOlaZAWQ9--qoyDqDXVefHj0jyCjcoxXulD39"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index f54b9563d4b2d..e275bd59d13ca 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -221,7 +221,7 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[cn4exTOUtxytuLkQAAABeRnR_mY][{_tsid=_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyP6Vt9ULEHW_gNYfd2t0r5Rnf6-L1KQ}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[cn4exTOUtxytuLkQAAABeRnR_mY][{_tsid=jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: @@ -240,9 +240,9 @@ ids query: values: ["cn4exTOUtxytuLkQAAABeRnR_mY", "cZZNs4NdV58ePSPIAAABeRnSA5M"] sort: ["@timestamp"] - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "cn4exWfgWRhamteRAAABeRnR_mY"} + - match: {hits.hits.0._id: "cn4exQbLKJ5tiyPoAAABeRnR_mY"} - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434595272]} - - match: {hits.hits.1._id: "cZZNs6__M5OgAaQpAAABeRnSA5M"} + - match: {hits.hits.1._id: "cZZNs9n0oXE_CDYsAAABeRnSA5M"} - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [2012916202]} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index a8ba6a6a6fd18..5c43642554b63 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -148,5 +148,5 @@ teardown: - match: {hits.total.value: 1} - match: {hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507} - - match: {hits.hits.0.fields._tsid: [{ _tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g" }]} + - match: {hits.hits.0.fields._tsid: [{ _tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g" }]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index 69ac13d66ef14..dc722ef3d2bea 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -184,7 +184,7 @@ fetch a tag: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}]} --- aggregate a dimension: @@ -281,9 +281,9 @@ aggregate a tag: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyP6Vt9ULEHW_gNYfd2t0r5Rnf6-L1KQ"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -318,7 +318,7 @@ sort by tsid: sort: [ "_tsid", "@timestamp" ] - match: {hits.total.value: 8} - - match: {hits.hits.0.sort: [{ _tsid : "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyP6Vt9ULEHW_gNYfd2t0r5Rnf6-L1KQ"}, 1619635803142]} - - match: {hits.hits.1.sort: [{ _tsid : "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyP6Vt9ULEHW_gNYfd2t0r5Rnf6-L1KQ"}, 1619635823142]} - - match: {hits.hits.4.sort: [{ _tsid : "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}, 1619635804467]} - - match: {hits.hits.7.sort: [{ _tsid : "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}, 1619635864467]} + - match: {hits.hits.0.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ"}, 1619635803142]} + - match: {hits.hits.1.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ"}, 1619635823142]} + - match: {hits.hits.4.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}, 1619635804467]} + - match: {hits.hits.7.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}, 1619635864467]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index 318441b35a2d7..e5d0152050b1d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -84,9 +84,9 @@ search an alias: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyP6Vt9ULEHW_gNYfd2t0r5Rnf6-L1KQ"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -128,10 +128,10 @@ index into alias: _key: asc - match: {hits.total.value: 12} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyP6Vt9ULEHW_gNYfd2t0r5Rnf6-L1KQ"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - match: {aggregations.tsids.buckets.2.key: {_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAC-lKWy--qoyP6Vt9ULEMwq1RsoCo9dtDZzvUhI5TA"}} + - match: {aggregations.tsids.buckets.2.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAC-lKWy--qoyMwq1RsoCo9dtDZzvUhI5TA"}} - match: {aggregations.tsids.buckets.2.doc_count: 4} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 17802979b16b8..816c6a647a404 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -48,7 +48,7 @@ add dimensions with put_mapping: - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "_pW31QsQ95YnAT0C6QOgFs_zuty0ZP6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAGlT_5_pW31QsQlhR27Th3c0p1ZhIq1uAyqA" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "95YnAT0C6QOgFs_zuty0ZAAAAAAAAAAAAAAAAAAAAAAGlT_5lhR27Th3c0p1ZhIq1uAyqA" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -96,7 +96,7 @@ add dimensions to no dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "_pW31QsQ95YnAT0C6QOgFs_zuty0ZP6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAGlT_5_pW31QsQlhR27Th3c0p1ZhIq1uAyqA" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "95YnAT0C6QOgFs_zuty0ZAAAAAAAAAAAAAAAAAAAAAAGlT_5lhR27Th3c0p1ZhIq1uAyqA" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -144,7 +144,7 @@ add dimensions to no dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "_pW31QsQ95YnAT0C6QOgFs_zuty0ZP6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAGlT_5_pW31QsQlhR27Th3c0p1ZhIq1uAyqA" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "95YnAT0C6QOgFs_zuty0ZAAAAAAAAAAAAAAAAAAAAAAGlT_5lhR27Th3c0p1ZhIq1uAyqA" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -196,7 +196,7 @@ add dimensions to some dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "_pW31QsQxhgWVhgKGfmIwHt-7D2B_v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-f6Vt9ULEHp7bXIK5rruMAWHvhNmhYw"} ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "xhgWVhgKGfmIwHt-7D2B_gAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-Xp7bXIK5rruMAWHvhNmhYw"} ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -247,5 +247,5 @@ add dimensions to some dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "_pW31QsQxhgWVhgKGfmIwHt-7D2B_v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-f6Vt9ULEHp7bXIK5rruMAWHvhNmhYw" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "xhgWVhgKGfmIwHt-7D2B_gAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-Xp7bXIK5rruMAWHvhNmhYw" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 4f554816f5b44..d8aca652411e1 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -64,10 +64,10 @@ keyword dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "_pW31QsQDh_r6QMn0c6jJhFGYOwd6P6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD_pW31QsQmAXfxM3_zbMMRzVn-ZChwA"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAA5ihHDmAXfxM3_zbMMRzVn-ZChwA"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "_pW31QsQDh_r6QMn0c6jJhFGYOwd6P6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh_pW31QsQVMfWLO-N2j8tEM_EPwLCbQ"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAB12oDhVMfWLO-N2j8tEM_EPwLCbQ"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -143,7 +143,7 @@ flattened dimension: - match: { hits.total.value: 8} - length: { aggregations.tsids.buckets: 4} - - match: { aggregations.tsids.buckets.0.key._tsid: "_pW31QsQP5PJzRH8-Gbp6mqjByZOmv6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAoaZ29eRDHR3kQx0fjCuTKddqA4f6Vt9ULEAfDLRt1ZH3XxmzpI4VE1d8" } + - match: { aggregations.tsids.buckets.0.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAAAoaZ29eRDHR3kQx0fjCuTKddqA4QfDLRt1ZH3XxmzpI4VE1d8" } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.0.key.deployment\.version\.minor: null } @@ -151,7 +151,7 @@ flattened dimension: - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "_pW31QsQP5PJzRH8-Gbp6mqjByZOmv6Vt9ULEAAAAAAAAAAAAAAAAAAAAACrnf7qeRDHR3kQx0eLDvTuOYoRw_6Vt9ULEBX9MyeUknhbUqQelj0fVGI" } + - match: { aggregations.tsids.buckets.1.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAACrnf7qeRDHR3kQx0eLDvTuOYoRwxX9MyeUknhbUqQelj0fVGI" } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.1.key.deployment\.version\.minor: null } @@ -159,7 +159,7 @@ flattened dimension: - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "_pW31QsQP5PJzRH8-Gbp6mqjByZOmv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADP1l1UeRDHR3kQx0fjCuTKddqA4f6Vt9ULEJbbw13wErAIwoyRvfPwBO0" } + - match: { aggregations.tsids.buckets.2.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAADP1l1UeRDHR3kQx0fjCuTKddqA4Zbbw13wErAIwoyRvfPwBO0" } - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.2.key.deployment\.version\.minor: null } @@ -167,7 +167,7 @@ flattened dimension: - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "_pW31QsQP5PJzRH8-Gbp6mqjByZOmv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADgKelzeRDHR3kQx0eLDvTuOYoRw_6Vt9ULECy-cFCrvRwidF9_OFLsEgY" } + - match: { aggregations.tsids.buckets.3.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAADgKelzeRDHR3kQx0eLDvTuOYoRwyy-cFCrvRwidF9_OFLsEgY" } - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.3.key.deployment\.version\.minor: null } @@ -247,11 +247,11 @@ flattened empty dimension: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 2 } - - match: { aggregations.tsids.buckets.0.key._tsid: "_pW31QsQDh_r6QMn0c6jJhFGYOwd6P6Vt9ULEAAAAAAAAAAAAAAAAAAAAAA5ihHD_pW31QsQmAXfxM3_zbMMRzVn-ZChwA" } + - match: { aggregations.tsids.buckets.0.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAA5ihHDmAXfxM3_zbMMRzVn-ZChwA" } - match: { aggregations.tsids.buckets.0.doc_count: 4 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "_pW31QsQDh_r6QMn0c6jJhFGYOwd6P6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh_pW31QsQVMfWLO-N2j8tEM_EPwLCbQ" } + - match: { aggregations.tsids.buckets.1.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAB12oDhVMfWLO-N2j8tEM_EPwLCbQ" } - match: { aggregations.tsids.buckets.1.doc_count: 4 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.22, error: 0.01 }} @@ -327,42 +327,42 @@ flattened field missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 6 } - - match: { aggregations.tsids.buckets.0.key._tsid: "_pW31QsQs0XCB0MdYlNxY9PEJqOWhv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADxTXebeRDHRzmKEcP-lbfVCxDP-9PVa-RfUDC6eRn9DO3J" } + - match: { aggregations.tsids.buckets.0.key._tsid: "s0XCB0MdYlNxY9PEJqOWhgAAAAAAAAAAAAAAAAAAAADxTXebeRDHRzmKEcPP-9PVa-RfUDC6eRn9DO3J" } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "_pW31QsQs0XCB0MdYlNxY9PEJqOWhv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADxTXebeRDHR3XagOH-lbfVCxAZbEZUQPevXNiuufZwN2d1" } + - match: { aggregations.tsids.buckets.1.key._tsid: "s0XCB0MdYlNxY9PEJqOWhgAAAAAAAAAAAAAAAAAAAADxTXebeRDHR3XagOEZbEZUQPevXNiuufZwN2d1" } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.1.doc_count: 1 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "_pW31QsQWPttUWoKR4uM6b9jH-_EXv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADxTXebKGmdvXkQx0d12oDh_pW31QsQ1FENdo5JPhSC9XstiiFMZw" } + - match: { aggregations.tsids.buckets.2.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXebKGmdvXkQx0d12oDh1FENdo5JPhSC9XstiiFMZw" } - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.2.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.2.doc_count: 1 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "_pW31QsQWPttUWoKR4uM6b9jH-_EXv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADxTXebq53-6nkQx0c5ihHD_pW31QsQi5nFHahYKUKfgq8xHVO8cA" } + - match: { aggregations.tsids.buckets.3.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXebq53-6nkQx0c5ihHDi5nFHahYKUKfgq8xHVO8cA" } - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.3.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.4.key._tsid: "_pW31QsQWPttUWoKR4uM6b9jH-_EXv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADxTXebz9ZdVHkQx0d12oDh_pW31QsQ-nmBYp5rUkkaiyO1h0pD9Q" } + - match: { aggregations.tsids.buckets.4.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXebz9ZdVHkQx0d12oDh-nmBYp5rUkkaiyO1h0pD9Q" } - match: { aggregations.tsids.buckets.4.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.4.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.4.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.4.doc_count: 2 } - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.5.key._tsid: "_pW31QsQWPttUWoKR4uM6b9jH-_EXv6Vt9ULEAAAAAAAAAAAAAAAAAAAAADxTXeb4Cnpc3kQx0c5ihHD_pW31QsQRLXz7VAY03chH7T4JK6g9g" } + - match: { aggregations.tsids.buckets.5.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXeb4Cnpc3kQx0c5ihHDRLXz7VAY03chH7T4JK6g9g" } - match: { aggregations.tsids.buckets.5.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.5.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.5.key.deployment\.version\.major: null } @@ -445,19 +445,19 @@ flattened field misspelled routing path field: - match: { hits.total.value: 6 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key._tsid: "_pW31QsQVpe7uQk6zUIFfFcpASZ8Y_6Vt9ULEAAAAAAAAAAAAAAAAAAAAAAoaZ29_pW31QsQ6X1JYjFYp22EdW9Qe7Fh3Q" } + - match: { aggregations.tsids.buckets.0.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAAAoaZ296X1JYjFYp22EdW9Qe7Fh3Q" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "_pW31QsQVpe7uQk6zUIFfFcpASZ8Y_6Vt9ULEAAAAAAAAAAAAAAAAAAAAACrnf7q_pW31QsQTp_Y-T2FMaY2THS8Sv5rUQ" } + - match: { aggregations.tsids.buckets.1.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAACrnf7qTp_Y-T2FMaY2THS8Sv5rUQ" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "_pW31QsQVpe7uQk6zUIFfFcpASZ8Y_6Vt9ULEAAAAAAAAAAAAAAAAAAAAADP1l1U_pW31QsQvSkqmNP15820DPPjhXpE7w" } + - match: { aggregations.tsids.buckets.2.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAADP1l1UvSkqmNP15820DPPjhXpE7w" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "_pW31QsQVpe7uQk6zUIFfFcpASZ8Y_6Vt9ULEAAAAAAAAAAAAAAAAAAAAADgKelz_pW31QsQ59jWtAPPEZOFAL6Fp-tkHA" } + - match: { aggregations.tsids.buckets.3.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAADgKelz59jWtAPPEZOFAL6Fp-tkHA" } - match: { aggregations.tsids.buckets.3.doc_count: 1 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} @@ -529,10 +529,10 @@ long dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "_pW31QsQXNiiZwxkczx2lrb8CBwni_6Vt9ULEAAAAAAAAAAAAAAAAAAAAABDgKYd73outv6Vt9ULEBnwPdwtIAd7vn-6M1jNT1s"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "XNiiZwxkczx2lrb8CBwniwAAAAAAAAAAAAAAAAAAAABDgKYd73outhnwPdwtIAd7vn-6M1jNT1s"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "_pW31QsQXNiiZwxkczx2lrb8CBwni_6Vt9ULEAAAAAAAAAAAAAAAAAAAAACtm41v73outv6Vt9ULELmJNwhM2k9uEB39QGAP8PY"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "XNiiZwxkczx2lrb8CBwniwAAAAAAAAAAAAAAAAAAAACtm41v73outrmJNwhM2k9uEB39QGAP8PY"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -604,10 +604,10 @@ ip dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: { _tsid: "_pW31QsQWlq2NkhwVgBD7x8-uwLcHf6Vt9ULEAAAAAAAAAAAAAAAAAAAAADYzb-v73outv6Vt9ULEBko4DlimDxhOKUBMl0caik"}} + - match: {aggregations.tsids.buckets.0.key: { _tsid: "Wlq2NkhwVgBD7x8-uwLcHQAAAAAAAAAAAAAAAAAAAADYzb-v73outhko4DlimDxhOKUBMl0caik"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: { _tsid: "_pW31QsQWlq2NkhwVgBD7x8-uwLcHf6Vt9ULEAAAAAAAAAAAAAAAAAAAAADaaOFK73outv6Vt9ULEJocP_SSPwvd1Uk-BTEqof0"}} + - match: {aggregations.tsids.buckets.1.key: { _tsid: "Wlq2NkhwVgBD7x8-uwLcHQAAAAAAAAAAAAAAAAAAAADaaOFK73outpocP_SSPwvd1Uk-BTEqof0"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index 015b7fbfd3d85..154749d64f022 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -123,7 +123,7 @@ shrink: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}]} --- clone: @@ -147,4 +147,4 @@ clone: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "_pW31QsQjsPn2w3PUBDHCdUJaEUx2v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyP6Vt9ULEMMN4bdnT8ZqELp_jWAw25g"}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}]} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 412a0193025c8..6cda6fd325b44 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -35,7 +35,6 @@ import java.io.IOException; import java.net.InetAddress; import java.time.ZoneId; -import java.util.Arrays; import java.util.Base64; import java.util.Collections; import java.util.Comparator; @@ -236,33 +235,39 @@ public BytesReference build() throws IOException { */ public BytesReference similarityHash() throws IOException { // NOTE: hash all dimension field names + int numberOfDimensions = Math.min(MAX_DIMENSIONS, dimensions.size()); + byte[] tsidHash = new byte[16 + 16 + 16 + 4 * numberOfDimensions]; + int dimensionValuesLength = 0; final StringBuilder sb = new StringBuilder(); for (final DimensionDataHolder dimension : dimensions) { - final BytesRef bytesRef = dimension.value.toBytesRef(); - dimensionValuesLength += (bytesRef.length - bytesRef.offset); + final BytesRef dimensionValueBytesRef = dimension.value.toBytesRef(); + dimensionValuesLength += (dimensionValueBytesRef.length - dimensionValueBytesRef.offset); sb.append(dimension.name); } - final BytesReference dimensionNamesHash = hash128(new BytesRef(sb.toString())); + final BytesRef dimensionNamesHash = hash128(new BytesRef(sb.toString())); + + int tsidHashIndex = 0; + System.arraycopy(dimensionNamesHash.bytes, dimensionNamesHash.offset, tsidHash, tsidHashIndex, dimensionNamesHash.length); + tsidHashIndex += dimensionNamesHash.length; // NOTE: hash all metric field names sb.setLength(0); for (final String metric : metrics) { sb.append(metric); } - final BytesReference metricFieldsHash = hash128(new BytesRef(sb.toString())); + final BytesRef metricFieldsHash = hash128(new BytesRef(sb.toString())); + System.arraycopy(metricFieldsHash.bytes, metricFieldsHash.offset, tsidHash, tsidHashIndex, metricFieldsHash.length); + tsidHashIndex += metricFieldsHash.length; // NOTE: catenate all dimension value hashes up to names certain number of dimensions - int numberOfDimensions = Math.min(MAX_DIMENSIONS, dimensions.size()); - int buf1Index = 0; - final byte[] buf1 = new byte[4 * numberOfDimensions]; + int tsidHashStartIndex = tsidHashIndex; for (final DimensionDataHolder dimension : dimensions) { - if (buf1Index >= buf1.length) break; + if ((tsidHashIndex - tsidHashStartIndex) >= 4 * numberOfDimensions) break; final BytesRef dimensionValueBytesRef = dimension.value().toBytesRef(); - ByteUtils.writeIntLE(StringHelper.murmurhash3_x86_32(dimensionValueBytesRef, 0), buf1, buf1Index); - buf1Index += 4; + ByteUtils.writeIntLE(StringHelper.murmurhash3_x86_32(dimensionValueBytesRef, 0), tsidHash, tsidHashIndex); + tsidHashIndex += 4; } - final BytesRef valueByValue = new BytesRef(buf1, 0, buf1.length); // NOTE: hash all dimension field allValues int buf2Index = 0; @@ -278,42 +283,23 @@ public BytesReference similarityHash() throws IOException { ); buf2Index += (dimensionValueBytesRef.length - dimensionValueBytesRef.offset); } - final BytesReference dimensionValuesHash = hash128(new BytesRef(buf2, 0, buf2.length)); - - final BytesRef names = dimensionNamesHash.toBytesRef(); - final BytesRef metrics = metricFieldsHash.toBytesRef(); - final BytesRef allValues = dimensionValuesHash.toBytesRef(); - int tsidLength = names.length + metrics.length + valueByValue.length + allValues.length; - byte[] tsid = new byte[tsidLength]; - int tsidIndex = 0; - System.arraycopy(names.bytes, names.offset, tsid, tsidIndex, names.length); - tsidIndex += names.length; - System.arraycopy(metrics.bytes, metrics.offset, tsid, tsidIndex, metrics.length); - tsidIndex += metrics.length; - System.arraycopy(valueByValue.bytes, valueByValue.offset, tsid, tsidIndex, valueByValue.length); - tsidIndex += valueByValue.length; - System.arraycopy(allValues.bytes, allValues.offset, tsid, tsidIndex, allValues.length); - tsidIndex += allValues.length; - assert tsidIndex == tsidLength; + final BytesRef dimensionValuesHash = hash128(new BytesRef(buf2, 0, buf2.length)); + System.arraycopy(dimensionValuesHash.bytes, dimensionValuesHash.offset, tsidHash, tsidHashIndex, dimensionValuesHash.length); + try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(TSID_HASH_SENTINEL); - out.writeBytesRef(new BytesRef(tsid, 0, tsidLength)); + out.writeBytesRef(new BytesRef(tsidHash, 0, tsidHash.length)); return out.bytes(); } } - private BytesReference hash128(final BytesRef value) throws IOException { + private BytesRef hash128(final BytesRef value) throws IOException { byte[] buffer = new byte[16]; final MurmurHash3.Hash128 hash128 = new MurmurHash3.Hash128(); MurmurHash3.hash128(value.bytes, value.offset, value.length, 0, hash128); ByteUtils.writeLongLE(hash128.h1, buffer, 0); ByteUtils.writeLongLE(hash128.h2, buffer, 8); - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(TSID_HASH_SENTINEL); - final BytesRef hashBytesRef = new BytesRef(Arrays.copyOfRange(buffer, 0, buffer.length)); - out.writeBytesRef(hashBytesRef); - return out.bytes(); - } + return new BytesRef(buffer); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index ffaf797f5d84d..46f60490c32b6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -78,10 +78,7 @@ public void testEnabledInTimeSeriesMode() throws Exception { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100).field("c", 500)); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry( - "_tsid", - "_pW31QsQLhvtFuoRi5Ot1FKbAade5v6Vt9ULEAAAAAAAAAAAAAAAAAAAAADuDjjxPwT2Ov6Vt9ULEMF14BMjwbnf88_kkYIqqxM" - ) + matchesMap().entry("_tsid", "LhvtFuoRi5Ot1FKbAade5gAAAAAAAAAAAAAAAAAAAADuDjjxPwT2OsF14BMjwbnf88_kkYIqqxM") ); } @@ -128,10 +125,7 @@ public void testStrings() throws IOException { ); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry( - "_tsid", - "_pW31QsQ-uh8SqJqlbcmNjsxQifzOf6Vt9ULEAAAAAAAAAAAAAAAAAAAAACv9houtA-WtP6Vt9ULEP9c3WHBQBL_APxECsSTB9g" - ) + matchesMap().entry("_tsid", "-uh8SqJqlbcmNjsxQifzOQAAAAAAAAAAAAAAAAAAAACv9houtA-WtP9c3WHBQBL_APxECsSTB9g") ); } @@ -147,13 +141,7 @@ public void testUnicodeKeys() throws IOException { Map tsid = TimeSeriesIdFieldMapper.decodeTsid( new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes) ); - assertMap( - tsid, - matchesMap().entry( - "_tsid", - "_pW31QsQJp7d1TeAbnqXtvYiILWo7v6Vt9ULEAAAAAAAAAAAAAAAAAAAAACejAXjDKlQ6v6Vt9ULEAYFWdDsVn7JHKYuKWckZQA" - ) - ); + assertMap(tsid, matchesMap().entry("_tsid", "Jp7d1TeAbnqXtvYiILWo7gAAAAAAAAAAAAAAAAAAAACejAXjDKlQ6gYFWdDsVn7JHKYuKWckZQA")); } public void testKeywordTooLong() throws IOException { @@ -164,7 +152,7 @@ public void testKeywordTooLong() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "_pW31QsQiXhZ9mVVVYVaiQ5RSDq15v6Vt9ULEAAAAAAAAAAAAAAAAAAAAAC9ZDUo_pW31QsQhD-XZN6wU-KM0X_aiyysUQ") + matchesMap().entry("_tsid", "iXhZ9mVVVYVaiQ5RSDq15gAAAAAAAAAAAAAAAAAAAAC9ZDUohD-XZN6wU-KM0X_aiyysUQ") ); } @@ -177,7 +165,7 @@ public void testKeywordTooLongUtf8() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "_pW31QsQiXhZ9mVVVYVaiQ5RSDq15v6Vt9ULEAAAAAAAAAAAAAAAAAAAAABM43f6_pW31QsQUqNsQMp1y6DdXAgTdWKAHg") + matchesMap().entry("_tsid", "iXhZ9mVVVYVaiQ5RSDq15gAAAAAAAAAAAAAAAAAAAABM43f6UqNsQMp1y6DdXAgTdWKAHg") ); } @@ -218,10 +206,7 @@ public void testLong() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry( - "_tsid", - "_pW31QsQy2oH44MiIyaUP_cSq588Qf6Vt9ULEAAAAAAAAAAAAAAAAAAAAACtm41vSrpJy91wa2D-lbfVCxBM5cSB4LNLdZ9rW4UIsHrm" - ) + matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJy91wa2BM5cSB4LNLdZ9rW4UIsHrm") ); } @@ -275,10 +260,7 @@ public void testInteger() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry( - "_tsid", - "_pW31QsQy2oH44MiIyaUP_cSq588Qf6Vt9ULEAAAAAAAAAAAAAAAAAAAAACtm41vSrpJy9XyEVD-lbfVCxBvG-5-7IUnwc1jT9EFsPcy" - ) + matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJy9XyEVBvG-5-7IUnwc1jT9EFsPcy") ); } @@ -336,10 +318,7 @@ public void testShort() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry( - "_tsid", - "_pW31QsQy2oH44MiIyaUP_cSq588Qf6Vt9ULEAAAAAAAAAAAAAAAAAAAAACtm41vSrpJyyYAWY_-lbfVCxBnXXrTqLOmwy6CMvTbNEBr" - ) + matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJyyYAWY9nXXrTqLOmwy6CMvTbNEBr") ); } @@ -397,10 +376,7 @@ public void testByte() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry( - "_tsid", - "_pW31QsQy2oH44MiIyaUP_cSq588Qf6Vt9ULEAAAAAAAAAAAAAAAAAAAAACtm41vSrpJyygrqof-lbfVCxAepXyuslV4c5JCMrunm5oE" - ) + matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJyygrqocepXyuslV4c5JCMrunm5oE") ); } @@ -458,10 +434,7 @@ public void testIp() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry( - "_tsid", - "_pW31QsQy2oH44MiIyaUP_cSq588Qf6Vt9ULEAAAAAAAAAAAAAAAAAAAAAALrOlWSrpJy7PzTuz-lbfVCxBolHEPXKdw6u7TkGuW-CJ7" - ) + matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAAALrOlWSrpJy7PzTuxolHEPXKdw6u7TkGuW-CJ7") ); } @@ -499,7 +472,7 @@ public void testVeryLarge() throws IOException { TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), matchesMap().entry( "_tsid", - "_pW31QsQVuMQB6AxmJbsu4Fzii310f6Vt9ULEAAAAAAAAAAAAAAAAAAAAACv9houWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk_6Vt9ULEOAOK-X4tmuBNmIlX0uEX_s" + "VuMQB6AxmJbsu4Fzii310QAAAAAAAAAAAAAAAAAAAACv9houWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk-AOK-X4tmuBNmIlX0uEX_s" ) ); } From 36edec56ef86156ebdd6d2607a6958afa1ed4959 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 19 Oct 2023 13:03:40 +0200 Subject: [PATCH 019/125] fix: optimize hash128 --- .../index/mapper/TimeSeriesIdFieldMapper.java | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 6cda6fd325b44..b8850770a6457 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -236,6 +236,7 @@ public BytesReference build() throws IOException { public BytesReference similarityHash() throws IOException { // NOTE: hash all dimension field names int numberOfDimensions = Math.min(MAX_DIMENSIONS, dimensions.size()); + int tsidHashIndex = 0; byte[] tsidHash = new byte[16 + 16 + 16 + 4 * numberOfDimensions]; int dimensionValuesLength = 0; @@ -245,20 +246,14 @@ public BytesReference similarityHash() throws IOException { dimensionValuesLength += (dimensionValueBytesRef.length - dimensionValueBytesRef.offset); sb.append(dimension.name); } - final BytesRef dimensionNamesHash = hash128(new BytesRef(sb.toString())); - - int tsidHashIndex = 0; - System.arraycopy(dimensionNamesHash.bytes, dimensionNamesHash.offset, tsidHash, tsidHashIndex, dimensionNamesHash.length); - tsidHashIndex += dimensionNamesHash.length; + tsidHashIndex += hash128(new BytesRef(sb.toString()), tsidHash, tsidHashIndex); // NOTE: hash all metric field names sb.setLength(0); for (final String metric : metrics) { sb.append(metric); } - final BytesRef metricFieldsHash = hash128(new BytesRef(sb.toString())); - System.arraycopy(metricFieldsHash.bytes, metricFieldsHash.offset, tsidHash, tsidHashIndex, metricFieldsHash.length); - tsidHashIndex += metricFieldsHash.length; + tsidHashIndex += hash128(new BytesRef(sb.toString()), tsidHash, tsidHashIndex); // NOTE: catenate all dimension value hashes up to names certain number of dimensions int tsidHashStartIndex = tsidHashIndex; @@ -283,8 +278,7 @@ public BytesReference similarityHash() throws IOException { ); buf2Index += (dimensionValueBytesRef.length - dimensionValueBytesRef.offset); } - final BytesRef dimensionValuesHash = hash128(new BytesRef(buf2, 0, buf2.length)); - System.arraycopy(dimensionValuesHash.bytes, dimensionValuesHash.offset, tsidHash, tsidHashIndex, dimensionValuesHash.length); + tsidHashIndex += hash128(new BytesRef(buf2, 0, buf2.length), tsidHash, tsidHashIndex); try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(TSID_HASH_SENTINEL); @@ -293,13 +287,12 @@ public BytesReference similarityHash() throws IOException { } } - private BytesRef hash128(final BytesRef value) throws IOException { - byte[] buffer = new byte[16]; + private int hash128(final BytesRef value, final byte[] buffer, int bufferIndex) throws IOException { final MurmurHash3.Hash128 hash128 = new MurmurHash3.Hash128(); MurmurHash3.hash128(value.bytes, value.offset, value.length, 0, hash128); - ByteUtils.writeLongLE(hash128.h1, buffer, 0); - ByteUtils.writeLongLE(hash128.h2, buffer, 8); - return new BytesRef(buffer); + ByteUtils.writeLongLE(hash128.h1, buffer, bufferIndex); + ByteUtils.writeLongLE(hash128.h2, buffer, bufferIndex + 8); + return 16; } @Override From b606182dbcdc46e61b6b06406d751a0b58ee1f9a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 19 Oct 2023 13:04:07 +0200 Subject: [PATCH 020/125] fix: some tsid values in tests --- .../test/tsdb/140_routing_path.yml | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 9ba89f803bf15..f1824021d2378 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -70,25 +70,25 @@ missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } + - match: { aggregations.tsids.buckets.0.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6Oix0Ruaua4Wo_hP7e-sEqo5ihHDmAXfxM3_zbMMRzVn-ZChwA" } - match: { aggregations.tsids.buckets.0.key.tag: null } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.15, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } + - match: { aggregations.tsids.buckets.1.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6Oix0Ruaua4Wo_hP7e-sEqp12oDhVMfWLO-N2j8tEM_EPwLCbQ" } - match: { aggregations.tsids.buckets.1.key.tag: null } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.69, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.15, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.2.key.tag: "first" } + - match: { aggregations.tsids.buckets.2.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqqaET_vOYoRw8zJD_nynQNV36JwKq7uZCw" } + - match: { aggregations.tsids.buckets.2.key.tag: null } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.30, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.3.key.tag: "second" } + - match: { aggregations.tsids.buckets.3.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqrcUWJEddqA4Vma0dpEIEx4FPmEt0nD4ck" } + - match: { aggregations.tsids.buckets.3.key.tag: null } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.70, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 7.30, error: 0.01 }} --- missing dimension on routing path field: @@ -201,12 +201,12 @@ multi-value routing path field: - match: {hits.total.value: 4} - length: {aggregations.tsids.buckets: 2} - - match: {aggregations.tsids.buckets.0.key.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: {aggregations.tsids.buckets.0.key.tag: "first" } + - match: {aggregations.tsids.buckets.0.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqqaET_vOYoRw8zJD_nynQNV36JwKq7uZCw" } + - match: {aggregations.tsids.buckets.0.key.tag: null } - match: {aggregations.tsids.buckets.0.doc_count: 2 } - - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 7.30, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.1.key.tag: "second" } + - match: { aggregations.tsids.buckets.1.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqrcUWJEddqA4Vma0dpEIEx4FPmEt0nD4ck" } + - match: { aggregations.tsids.buckets.1.key.tag: null } - match: {aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 6.70, error: 0.01 }} + - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} From e38babda6778e757b8e6a539d6a4d946ea4d6466 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 19 Oct 2023 17:23:01 +0200 Subject: [PATCH 021/125] fix: update tsid hash reducing memory allocations --- .../test/aggregations/time_series.yml | 10 ++-- .../test/data_stream/150_tsdb.yml | 6 +-- .../test/tsdb/140_routing_path.yml | 12 ++--- .../test/tsdb/25_id_generation.yml | 6 +-- .../rest-api-spec/test/tsdb/30_snapshot.yml | 2 +- .../rest-api-spec/test/tsdb/40_search.yml | 14 ++--- .../rest-api-spec/test/tsdb/50_alias.yml | 10 ++-- .../test/tsdb/60_add_dimensions.yml | 10 ++-- .../test/tsdb/70_dimension_types.yml | 44 ++++++++-------- .../test/tsdb/80_index_resize.yml | 4 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 52 ++++++++----------- .../index/mapper/KeywordFieldMapperTests.java | 9 +++- .../mapper/TimeSeriesIdFieldMapperTests.java | 22 ++++---- 13 files changed, 100 insertions(+), 101 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index f5a8d99b55b2c..23b497c557518 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -76,7 +76,7 @@ setup: - match: { hits.total.value: 1 } - length: { aggregations: 1 } - - match: { aggregations.ts.buckets.0.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACv9houG2TKanX0IDDfq7mPr0SxJA" } } + - match: { aggregations.ts.buckets.0.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACv9hou4__k9oZfeombF5dLPwkqUA" } } - match: { aggregations.ts.buckets.0.doc_count: 1 } --- @@ -126,7 +126,7 @@ setup: size: 1 - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACKf_6WEKY54y0GuGgKb5oO6t-5YQ" } } + - match: { aggregations.ts.buckets.0.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACKf_6WZzMy4zeMjdIBoQC3xHGkjg" } } - do: search: @@ -144,9 +144,9 @@ setup: size: 3 - length: { aggregations.ts.buckets: 3 } - - match: { aggregations.ts.buckets.0.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACKf_6WEKY54y0GuGgKb5oO6t-5YQ" } } - - match: { aggregations.ts.buckets.1.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACv9houG2TKanX0IDDfq7mPr0SxJA" } } - - match: { aggregations.ts.buckets.2.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAAC5AZB2QuYbdK0MxkFW7lg9z6q4rA" } } + - match: { aggregations.ts.buckets.0.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACKf_6WZzMy4zeMjdIBoQC3xHGkjg" } } + - match: { aggregations.ts.buckets.1.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACv9hou4__k9oZfeombF5dLPwkqUA" } } + - match: { aggregations.ts.buckets.2.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAAC5AZB2-M4K_VYiHrKDjUNX3zHVkw" } } --- "Score test filter some": diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 1ab73edce3add..4f4e92c0a0499 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -143,7 +143,7 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g" }]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" }]} --- "aggregate the tsid": @@ -164,9 +164,9 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index f1824021d2378..381ad0011a8fa 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -70,22 +70,22 @@ missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6Oix0Ruaua4Wo_hP7e-sEqo5ihHDmAXfxM3_zbMMRzVn-ZChwA" } + - match: { aggregations.tsids.buckets.0.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6Oix0Ruaua4Wo_hP7e-sEqo5ihHDIG1DaFBdVI_fYOQvJgKOvg" } - match: { aggregations.tsids.buckets.0.key.tag: null } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6Oix0Ruaua4Wo_hP7e-sEqp12oDhVMfWLO-N2j8tEM_EPwLCbQ" } + - match: { aggregations.tsids.buckets.1.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6Oix0Ruaua4Wo_hP7e-sEqp12oDh9NI69d0Kk5TQ6CAdewYP5A" } - match: { aggregations.tsids.buckets.1.key.tag: null } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.15, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqqaET_vOYoRw8zJD_nynQNV36JwKq7uZCw" } + - match: { aggregations.tsids.buckets.2.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqqaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } - match: { aggregations.tsids.buckets.2.key.tag: null } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqrcUWJEddqA4Vma0dpEIEx4FPmEt0nD4ck" } + - match: { aggregations.tsids.buckets.3.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqrcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } - match: { aggregations.tsids.buckets.3.key.tag: null } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 7.30, error: 0.01 }} @@ -201,12 +201,12 @@ multi-value routing path field: - match: {hits.total.value: 4} - length: {aggregations.tsids.buckets: 2} - - match: {aggregations.tsids.buckets.0.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqqaET_vOYoRw8zJD_nynQNV36JwKq7uZCw" } + - match: {aggregations.tsids.buckets.0.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqqaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } - match: {aggregations.tsids.buckets.0.key.tag: null } - match: {aggregations.tsids.buckets.0.doc_count: 2 } - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqrcUWJEddqA4Vma0dpEIEx4FPmEt0nD4ck" } + - match: { aggregations.tsids.buckets.1.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqrcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } - match: { aggregations.tsids.buckets.1.key.tag: null } - match: {aggregations.tsids.buckets.1.doc_count: 2 } - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index e275bd59d13ca..c3338b174708a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -221,7 +221,7 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[cn4exTOUtxytuLkQAAABeRnR_mY][{_tsid=jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[cn4exTOUtxytuLkQAAABeRnR_mY][{_tsid=jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: @@ -240,9 +240,9 @@ ids query: values: ["cn4exTOUtxytuLkQAAABeRnR_mY", "cZZNs4NdV58ePSPIAAABeRnSA5M"] sort: ["@timestamp"] - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "cn4exQbLKJ5tiyPoAAABeRnR_mY"} + - match: {hits.hits.0._id: "cn4exafToJ7Wv1jaAAABeRnR_mY"} - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434595272]} - - match: {hits.hits.1._id: "cZZNs9n0oXE_CDYsAAABeRnSA5M"} + - match: {hits.hits.1._id: "cZZNs8_QMMcY0B43AAABeRnSA5M"} - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [2012916202]} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 5c43642554b63..f32de68435cc6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -148,5 +148,5 @@ teardown: - match: {hits.total.value: 1} - match: {hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507} - - match: {hits.hits.0.fields._tsid: [{ _tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g" }]} + - match: {hits.hits.0.fields._tsid: [{ _tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" }]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index dc722ef3d2bea..8bfe2945b2a2f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -184,7 +184,7 @@ fetch a tag: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}]} --- aggregate a dimension: @@ -281,9 +281,9 @@ aggregate a tag: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -318,7 +318,7 @@ sort by tsid: sort: [ "_tsid", "@timestamp" ] - match: {hits.total.value: 8} - - match: {hits.hits.0.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ"}, 1619635803142]} - - match: {hits.hits.1.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ"}, 1619635823142]} - - match: {hits.hits.4.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}, 1619635804467]} - - match: {hits.hits.7.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}, 1619635864467]} + - match: {hits.hits.0.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}, 1619635803142]} + - match: {hits.hits.1.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}, 1619635823142]} + - match: {hits.hits.4.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}, 1619635804467]} + - match: {hits.hits.7.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}, 1619635864467]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index e5d0152050b1d..1fe9a277fc331 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -84,9 +84,9 @@ search an alias: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -128,10 +128,10 @@ index into alias: _key: asc - match: {hits.total.value: 12} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyHW_gNYfd2t0r5Rnf6-L1KQ"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - match: {aggregations.tsids.buckets.2.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAC-lKWy--qoyMwq1RsoCo9dtDZzvUhI5TA"}} + - match: {aggregations.tsids.buckets.2.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAC-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"}} - match: {aggregations.tsids.buckets.2.doc_count: 4} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 816c6a647a404..917512d0d07fe 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -48,7 +48,7 @@ add dimensions with put_mapping: - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "95YnAT0C6QOgFs_zuty0ZAAAAAAAAAAAAAAAAAAAAAAGlT_5lhR27Th3c0p1ZhIq1uAyqA" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "95YnAT0C6QOgFs_zuty0ZAAAAAAAAAAAAAAAAAAAAAAGlT_5e6_NYGOZWULpmMG9IAlZlA" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -96,7 +96,7 @@ add dimensions to no dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "95YnAT0C6QOgFs_zuty0ZAAAAAAAAAAAAAAAAAAAAAAGlT_5lhR27Th3c0p1ZhIq1uAyqA" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "95YnAT0C6QOgFs_zuty0ZAAAAAAAAAAAAAAAAAAAAAAGlT_5e6_NYGOZWULpmMG9IAlZlA" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -144,7 +144,7 @@ add dimensions to no dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "95YnAT0C6QOgFs_zuty0ZAAAAAAAAAAAAAAAAAAAAAAGlT_5lhR27Th3c0p1ZhIq1uAyqA" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "95YnAT0C6QOgFs_zuty0ZAAAAAAAAAAAAAAAAAAAAAAGlT_5e6_NYGOZWULpmMG9IAlZlA" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -196,7 +196,7 @@ add dimensions to some dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "xhgWVhgKGfmIwHt-7D2B_gAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-Xp7bXIK5rruMAWHvhNmhYw"} ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "xhgWVhgKGfmIwHt-7D2B_gAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y"} ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -247,5 +247,5 @@ add dimensions to some dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "xhgWVhgKGfmIwHt-7D2B_gAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-Xp7bXIK5rruMAWHvhNmhYw" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "xhgWVhgKGfmIwHt-7D2B_gAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index d8aca652411e1..0a4edc403c3a7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -64,10 +64,10 @@ keyword dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAA5ihHDmAXfxM3_zbMMRzVn-ZChwA"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAA5ihHDIG1DaFBdVI_fYOQvJgKOvg"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAB12oDhVMfWLO-N2j8tEM_EPwLCbQ"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAB12oDh9NI69d0Kk5TQ6CAdewYP5A"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -143,7 +143,7 @@ flattened dimension: - match: { hits.total.value: 8} - length: { aggregations.tsids.buckets: 4} - - match: { aggregations.tsids.buckets.0.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAAAoaZ29eRDHR3kQx0fjCuTKddqA4QfDLRt1ZH3XxmzpI4VE1d8" } + - match: { aggregations.tsids.buckets.0.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAAAoaZ29eRDHR3kQx0fjCuTKddqA4R9ytcYdqdbcoJ8VBuvqFtQ" } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.0.key.deployment\.version\.minor: null } @@ -151,7 +151,7 @@ flattened dimension: - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAACrnf7qeRDHR3kQx0eLDvTuOYoRwxX9MyeUknhbUqQelj0fVGI" } + - match: { aggregations.tsids.buckets.1.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAACrnf7qeRDHR3kQx0eLDvTuOYoRw4EeUIKK1KGFPmxqCWQyQJE" } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.1.key.deployment\.version\.minor: null } @@ -159,7 +159,7 @@ flattened dimension: - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAADP1l1UeRDHR3kQx0fjCuTKddqA4Zbbw13wErAIwoyRvfPwBO0" } + - match: { aggregations.tsids.buckets.2.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAADP1l1UeRDHR3kQx0fjCuTKddqA4ai7CiSuCxA-PhBdLYOXkVY" } - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.2.key.deployment\.version\.minor: null } @@ -167,7 +167,7 @@ flattened dimension: - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAADgKelzeRDHR3kQx0eLDvTuOYoRwyy-cFCrvRwidF9_OFLsEgY" } + - match: { aggregations.tsids.buckets.3.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAADgKelzeRDHR3kQx0eLDvTuOYoRw6Wapg_P07YIhdV3NFJDjjE" } - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.3.key.deployment\.version\.minor: null } @@ -247,11 +247,11 @@ flattened empty dimension: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 2 } - - match: { aggregations.tsids.buckets.0.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAA5ihHDmAXfxM3_zbMMRzVn-ZChwA" } + - match: { aggregations.tsids.buckets.0.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAA5ihHDIG1DaFBdVI_fYOQvJgKOvg" } - match: { aggregations.tsids.buckets.0.doc_count: 4 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAB12oDhVMfWLO-N2j8tEM_EPwLCbQ" } + - match: { aggregations.tsids.buckets.1.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAB12oDh9NI69d0Kk5TQ6CAdewYP5A" } - match: { aggregations.tsids.buckets.1.doc_count: 4 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.22, error: 0.01 }} @@ -327,42 +327,42 @@ flattened field missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 6 } - - match: { aggregations.tsids.buckets.0.key._tsid: "s0XCB0MdYlNxY9PEJqOWhgAAAAAAAAAAAAAAAAAAAADxTXebeRDHRzmKEcPP-9PVa-RfUDC6eRn9DO3J" } + - match: { aggregations.tsids.buckets.0.key._tsid: "s0XCB0MdYlNxY9PEJqOWhgAAAAAAAAAAAAAAAAAAAADxTXebeRDHRzmKEcOUW9u_kzgC7pSzoi1utVcm" } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "s0XCB0MdYlNxY9PEJqOWhgAAAAAAAAAAAAAAAAAAAADxTXebeRDHR3XagOEZbEZUQPevXNiuufZwN2d1" } + - match: { aggregations.tsids.buckets.1.key._tsid: "s0XCB0MdYlNxY9PEJqOWhgAAAAAAAAAAAAAAAAAAAADxTXebeRDHR3XagOEjGQJfwIB2Q5kLDL76leH4" } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.1.doc_count: 1 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXebKGmdvXkQx0d12oDh1FENdo5JPhSC9XstiiFMZw" } + - match: { aggregations.tsids.buckets.2.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXebKGmdvXkQx0d12oDh16FtfvP_ObklCa7gXDcmdA" } - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.2.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.2.doc_count: 1 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXebq53-6nkQx0c5ihHDi5nFHahYKUKfgq8xHVO8cA" } + - match: { aggregations.tsids.buckets.3.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXebq53-6nkQx0c5ihHDU3PuzTI1mM98gQqqihDoZA" } - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.3.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.4.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXebz9ZdVHkQx0d12oDh-nmBYp5rUkkaiyO1h0pD9Q" } + - match: { aggregations.tsids.buckets.4.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXebz9ZdVHkQx0d12oDhi1SccpVnRAhpjCrWhqFjfQ" } - match: { aggregations.tsids.buckets.4.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.4.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.4.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.4.doc_count: 2 } - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.5.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXeb4Cnpc3kQx0c5ihHDRLXz7VAY03chH7T4JK6g9g" } + - match: { aggregations.tsids.buckets.5.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXeb4Cnpc3kQx0c5ihHDqaS5hHgLAm0E3XkHG7fdkQ" } - match: { aggregations.tsids.buckets.5.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.5.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.5.key.deployment\.version\.major: null } @@ -445,19 +445,19 @@ flattened field misspelled routing path field: - match: { hits.total.value: 6 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAAAoaZ296X1JYjFYp22EdW9Qe7Fh3Q" } + - match: { aggregations.tsids.buckets.0.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAAAoaZ29egRdyNeHXPSPghDVzguaRg" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAACrnf7qTp_Y-T2FMaY2THS8Sv5rUQ" } + - match: { aggregations.tsids.buckets.1.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAACrnf7qs9-VXFZ6jjZCbl_iiXSs7Q" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAADP1l1UvSkqmNP15820DPPjhXpE7w" } + - match: { aggregations.tsids.buckets.2.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAADP1l1UlmlXEQVNXrHpUvpn7by0jA" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAADgKelz59jWtAPPEZOFAL6Fp-tkHA" } + - match: { aggregations.tsids.buckets.3.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAADgKelz9WSJqzeYh7aza_7yxDXMZA" } - match: { aggregations.tsids.buckets.3.doc_count: 1 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} @@ -529,10 +529,10 @@ long dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "XNiiZwxkczx2lrb8CBwniwAAAAAAAAAAAAAAAAAAAABDgKYd73outhnwPdwtIAd7vn-6M1jNT1s"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "XNiiZwxkczx2lrb8CBwniwAAAAAAAAAAAAAAAAAAAABDgKYd73outpXn7LJV-gQfvlrec7NyMho"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "XNiiZwxkczx2lrb8CBwniwAAAAAAAAAAAAAAAAAAAACtm41v73outrmJNwhM2k9uEB39QGAP8PY"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "XNiiZwxkczx2lrb8CBwniwAAAAAAAAAAAAAAAAAAAACtm41v73outmoSTcFmfQBYjOjMaOWM5zs"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -604,10 +604,10 @@ ip dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: { _tsid: "Wlq2NkhwVgBD7x8-uwLcHQAAAAAAAAAAAAAAAAAAAADYzb-v73outhko4DlimDxhOKUBMl0caik"}} + - match: {aggregations.tsids.buckets.0.key: { _tsid: "Wlq2NkhwVgBD7x8-uwLcHQAAAAAAAAAAAAAAAAAAAADYzb-v73outosB-eDb_nzTrIVsJFVQR2c"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: { _tsid: "Wlq2NkhwVgBD7x8-uwLcHQAAAAAAAAAAAAAAAAAAAADaaOFK73outpocP_SSPwvd1Uk-BTEqof0"}} + - match: {aggregations.tsids.buckets.1.key: { _tsid: "Wlq2NkhwVgBD7x8-uwLcHQAAAAAAAAAAAAAAAAAAAADaaOFK73outkqpG8R65Gm4VUhpyuc11zw"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index 154749d64f022..ac7d72497f513 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -123,7 +123,7 @@ shrink: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}]} --- clone: @@ -147,4 +147,4 @@ clone: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyMMN4bdnT8ZqELp_jWAw25g"}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}]} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index b8850770a6457..8e3e4bc2767df 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -15,6 +15,7 @@ import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.hash.Murmur3Hasher; import org.elasticsearch.common.hash.MurmurHash3; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; @@ -34,6 +35,7 @@ import java.io.IOException; import java.net.InetAddress; +import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.util.Base64; import java.util.Collections; @@ -189,6 +191,10 @@ public static class TimeSeriesIdBuilder implements DocumentFields { private record DimensionDataHolder(String name, BytesReference value) {} + private static final int SEED = 0; + + private final Murmur3Hasher hasher = new Murmur3Hasher(SEED); + /** * A sorted set of the serialized values of dimension fields that will be used * for generating the _tsid field. The map will be used by {@link TimeSeriesIdFieldMapper} @@ -239,46 +245,33 @@ public BytesReference similarityHash() throws IOException { int tsidHashIndex = 0; byte[] tsidHash = new byte[16 + 16 + 16 + 4 * numberOfDimensions]; - int dimensionValuesLength = 0; - final StringBuilder sb = new StringBuilder(); + hasher.reset(); for (final DimensionDataHolder dimension : dimensions) { - final BytesRef dimensionValueBytesRef = dimension.value.toBytesRef(); - dimensionValuesLength += (dimensionValueBytesRef.length - dimensionValueBytesRef.offset); - sb.append(dimension.name); + hasher.update(dimension.name.getBytes(StandardCharsets.UTF_8)); } - tsidHashIndex += hash128(new BytesRef(sb.toString()), tsidHash, tsidHashIndex); + tsidHashIndex = hash128(tsidHash, tsidHashIndex); // NOTE: hash all metric field names - sb.setLength(0); + hasher.reset(); for (final String metric : metrics) { - sb.append(metric); + hasher.update(metric.getBytes(StandardCharsets.UTF_8)); } - tsidHashIndex += hash128(new BytesRef(sb.toString()), tsidHash, tsidHashIndex); + tsidHashIndex = hash128(tsidHash, tsidHashIndex); // NOTE: catenate all dimension value hashes up to names certain number of dimensions int tsidHashStartIndex = tsidHashIndex; for (final DimensionDataHolder dimension : dimensions) { if ((tsidHashIndex - tsidHashStartIndex) >= 4 * numberOfDimensions) break; - final BytesRef dimensionValueBytesRef = dimension.value().toBytesRef(); - ByteUtils.writeIntLE(StringHelper.murmurhash3_x86_32(dimensionValueBytesRef, 0), tsidHash, tsidHashIndex); + ByteUtils.writeIntLE(StringHelper.murmurhash3_x86_32(dimension.value.toBytesRef(), SEED), tsidHash, tsidHashIndex); tsidHashIndex += 4; } // NOTE: hash all dimension field allValues - int buf2Index = 0; - final byte[] buf2 = new byte[dimensionValuesLength]; + hasher.reset(); for (final DimensionDataHolder dimension : dimensions) { - final BytesRef dimensionValueBytesRef = dimension.value.toBytesRef(); - System.arraycopy( - dimensionValueBytesRef.bytes, - dimensionValueBytesRef.offset, - buf2, - buf2Index, - dimensionValueBytesRef.length - dimensionValueBytesRef.offset - ); - buf2Index += (dimensionValueBytesRef.length - dimensionValueBytesRef.offset); + hasher.update(dimension.value.array()); } - tsidHashIndex += hash128(new BytesRef(buf2, 0, buf2.length), tsidHash, tsidHashIndex); + tsidHashIndex += hash128(tsidHash, tsidHashIndex); try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(TSID_HASH_SENTINEL); @@ -287,12 +280,13 @@ public BytesReference similarityHash() throws IOException { } } - private int hash128(final BytesRef value, final byte[] buffer, int bufferIndex) throws IOException { - final MurmurHash3.Hash128 hash128 = new MurmurHash3.Hash128(); - MurmurHash3.hash128(value.bytes, value.offset, value.length, 0, hash128); - ByteUtils.writeLongLE(hash128.h1, buffer, bufferIndex); - ByteUtils.writeLongLE(hash128.h2, buffer, bufferIndex + 8); - return 16; + private int hash128(byte[] tsidHash, int tsidHashIndex) { + MurmurHash3.Hash128 namesHash = Murmur3Hasher.toHash128(hasher.digest()); + ByteUtils.writeLongLE(namesHash.h1, tsidHash, tsidHashIndex); + tsidHashIndex += 8; + ByteUtils.writeLongLE(namesHash.h2, tsidHash, tsidHashIndex); + tsidHashIndex += 8; + return tsidHashIndex; } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index ba963701d57f4..9d123378ae92b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -398,10 +398,15 @@ public void testDimensionExtraLongKeyword() throws IOException { Exception e = expectThrows( DocumentParsingException.class, () -> mapper.parse( - source(b -> b.field("field", randomAlphaOfLengthBetween(IndexWriter.MAX_TERM_LENGTH - 100, IndexWriter.MAX_TERM_LENGTH))) + source(b -> b.field("field", randomAlphaOfLengthBetween(IndexWriter.MAX_TERM_LENGTH, IndexWriter.MAX_TERM_LENGTH + 100))) + ) + ); + assertThat( + e.getCause().getMessage(), + containsString( + "Document contains at least one immense term in field=\"field\" (whose UTF8 encoding is longer than the max length 32766" ) ); - assertThat(e.getCause().getMessage(), equalTo("data stream timestamp field [@timestamp] is missing")); } public void testConfigureSimilarity() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 46f60490c32b6..5af2910e03b84 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -78,7 +78,7 @@ public void testEnabledInTimeSeriesMode() throws Exception { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100).field("c", 500)); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "LhvtFuoRi5Ot1FKbAade5gAAAAAAAAAAAAAAAAAAAADuDjjxPwT2OsF14BMjwbnf88_kkYIqqxM") + matchesMap().entry("_tsid", "LhvtFuoRi5Ot1FKbAade5gAAAAAAAAAAAAAAAAAAAADuDjjxPwT2Ol0UAzaqxX0Rpx75AkZHRdY") ); } @@ -125,7 +125,7 @@ public void testStrings() throws IOException { ); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "-uh8SqJqlbcmNjsxQifzOQAAAAAAAAAAAAAAAAAAAACv9houtA-WtP9c3WHBQBL_APxECsSTB9g") + matchesMap().entry("_tsid", "-uh8SqJqlbcmNjsxQifzOQAAAAAAAAAAAAAAAAAAAACv9houtA-WtALkiErAWD5o8PqlkO2CPtw") ); } @@ -141,7 +141,7 @@ public void testUnicodeKeys() throws IOException { Map tsid = TimeSeriesIdFieldMapper.decodeTsid( new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes) ); - assertMap(tsid, matchesMap().entry("_tsid", "Jp7d1TeAbnqXtvYiILWo7gAAAAAAAAAAAAAAAAAAAACejAXjDKlQ6gYFWdDsVn7JHKYuKWckZQA")); + assertMap(tsid, matchesMap().entry("_tsid", "Jp7d1TeAbnqXtvYiILWo7gAAAAAAAAAAAAAAAAAAAACejAXjDKlQ6oAVV6S1YhlDnYAgGZ4kqeg")); } public void testKeywordTooLong() throws IOException { @@ -152,7 +152,7 @@ public void testKeywordTooLong() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "iXhZ9mVVVYVaiQ5RSDq15gAAAAAAAAAAAAAAAAAAAAC9ZDUohD-XZN6wU-KM0X_aiyysUQ") + matchesMap().entry("_tsid", "iXhZ9mVVVYVaiQ5RSDq15gAAAAAAAAAAAAAAAAAAAAC9ZDUoCO2eynkD-lZbEGc5huKIYw") ); } @@ -165,7 +165,7 @@ public void testKeywordTooLongUtf8() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "iXhZ9mVVVYVaiQ5RSDq15gAAAAAAAAAAAAAAAAAAAABM43f6UqNsQMp1y6DdXAgTdWKAHg") + matchesMap().entry("_tsid", "iXhZ9mVVVYVaiQ5RSDq15gAAAAAAAAAAAAAAAAAAAABM43f6n6D4bFDIodLIvFw5j_1Vew") ); } @@ -206,7 +206,7 @@ public void testLong() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJy91wa2BM5cSB4LNLdZ9rW4UIsHrm") + matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJy91wa2A3yf66JQYgw9_3GPhrunFA") ); } @@ -260,7 +260,7 @@ public void testInteger() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJy9XyEVBvG-5-7IUnwc1jT9EFsPcy") + matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJy9XyEVAdPjbPnyFfNkppmgRrzARX") ); } @@ -318,7 +318,7 @@ public void testShort() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJyyYAWY9nXXrTqLOmwy6CMvTbNEBr") + matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJyyYAWY_JC1ss-P30niOYXA2x9kLq") ); } @@ -376,7 +376,7 @@ public void testByte() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJyygrqocepXyuslV4c5JCMrunm5oE") + matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJyygrqocRYcbJqQJZlIyTzrY9S9Nt") ); } @@ -434,7 +434,7 @@ public void testIp() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAAALrOlWSrpJy7PzTuxolHEPXKdw6u7TkGuW-CJ7") + matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAAALrOlWSrpJy7PzTuwgaqmbsw6Oz00Ir_8Cw9Xm") ); } @@ -472,7 +472,7 @@ public void testVeryLarge() throws IOException { TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), matchesMap().entry( "_tsid", - "VuMQB6AxmJbsu4Fzii310QAAAAAAAAAAAAAAAAAAAACv9houWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk-AOK-X4tmuBNmIlX0uEX_s" + "VuMQB6AxmJbsu4Fzii310QAAAAAAAAAAAAAAAAAAAACv9houWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk8VBv2rLPN9GYxF8e8uXojI" ) ); } From 28c131986752ce5dce9541a8e9328df2f6957be0 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 19 Oct 2023 17:31:03 +0200 Subject: [PATCH 022/125] fix: comment --- .../org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 8e3e4bc2767df..f0ead8ed77763 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -258,7 +258,7 @@ public BytesReference similarityHash() throws IOException { } tsidHashIndex = hash128(tsidHash, tsidHashIndex); - // NOTE: catenate all dimension value hashes up to names certain number of dimensions + // NOTE: concatenate all dimension value hashes up to a certain number of dimensions int tsidHashStartIndex = tsidHashIndex; for (final DimensionDataHolder dimension : dimensions) { if ((tsidHashIndex - tsidHashStartIndex) >= 4 * numberOfDimensions) break; From c5fa3ccce35cbb3f9b6addec9b1726d0b9fe994d Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 30 Oct 2023 10:44:44 +0100 Subject: [PATCH 023/125] fix: simplify tsid hashing --- .../index/mapper/TimeSeriesIdFieldMapper.java | 48 +++++-------------- 1 file changed, 13 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index f0ead8ed77763..ee1026fe38b28 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -11,16 +11,13 @@ import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.StringHelper; import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.hash.Murmur3Hasher; -import org.elasticsearch.common.hash.MurmurHash3; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.network.NetworkAddress; -import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.fielddata.FieldData; @@ -189,11 +186,9 @@ public static class TimeSeriesIdBuilder implements DocumentFields { public static final int MAX_DIMENSIONS = 512; - private record DimensionDataHolder(String name, BytesReference value) {} + private record DimensionDataHolder(BytesRef name, BytesRef value) {} - private static final int SEED = 0; - - private final Murmur3Hasher hasher = new Murmur3Hasher(SEED); + private final Murmur3Hasher tsidHasher = new Murmur3Hasher(0); /** * A sorted set of the serialized values of dimension fields that will be used @@ -220,8 +215,8 @@ public BytesReference build() throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(dimensions.size()); for (DimensionDataHolder entry : dimensions) { - out.writeBytesRef(new BytesRef(entry.name)); - entry.value.writeTo(out); + out.writeBytesRef(entry.name); + out.writeBytesRef(entry.value); } return out.bytes(); } @@ -242,53 +237,36 @@ public BytesReference build() throws IOException { public BytesReference similarityHash() throws IOException { // NOTE: hash all dimension field names int numberOfDimensions = Math.min(MAX_DIMENSIONS, dimensions.size()); - int tsidHashIndex = 0; - byte[] tsidHash = new byte[16 + 16 + 16 + 4 * numberOfDimensions]; - hasher.reset(); for (final DimensionDataHolder dimension : dimensions) { - hasher.update(dimension.name.getBytes(StandardCharsets.UTF_8)); + tsidHasher.update(dimension.name.bytes); } - tsidHashIndex = hash128(tsidHash, tsidHashIndex); // NOTE: hash all metric field names - hasher.reset(); for (final String metric : metrics) { - hasher.update(metric.getBytes(StandardCharsets.UTF_8)); + tsidHasher.update(metric.getBytes(StandardCharsets.UTF_8)); } - tsidHashIndex = hash128(tsidHash, tsidHashIndex); // NOTE: concatenate all dimension value hashes up to a certain number of dimensions - int tsidHashStartIndex = tsidHashIndex; + int dimensionCounter = 0; for (final DimensionDataHolder dimension : dimensions) { - if ((tsidHashIndex - tsidHashStartIndex) >= 4 * numberOfDimensions) break; - ByteUtils.writeIntLE(StringHelper.murmurhash3_x86_32(dimension.value.toBytesRef(), SEED), tsidHash, tsidHashIndex); - tsidHashIndex += 4; + if (dimensionCounter > numberOfDimensions) break; + tsidHasher.update(dimension.value.bytes); + dimensionCounter++; } // NOTE: hash all dimension field allValues - hasher.reset(); for (final DimensionDataHolder dimension : dimensions) { - hasher.update(dimension.value.array()); + tsidHasher.update(dimension.value.bytes); } - tsidHashIndex += hash128(tsidHash, tsidHashIndex); try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(TSID_HASH_SENTINEL); - out.writeBytesRef(new BytesRef(tsidHash, 0, tsidHash.length)); + out.writeBytesRef(new BytesRef(tsidHasher.digest())); return out.bytes(); } } - private int hash128(byte[] tsidHash, int tsidHashIndex) { - MurmurHash3.Hash128 namesHash = Murmur3Hasher.toHash128(hasher.digest()); - ByteUtils.writeLongLE(namesHash.h1, tsidHash, tsidHashIndex); - tsidHashIndex += 8; - ByteUtils.writeLongLE(namesHash.h2, tsidHash, tsidHashIndex); - tsidHashIndex += 8; - return tsidHashIndex; - } - @Override public void addKeywordDimension(String fieldName, BytesRef utf8Value) { try (BytesStreamOutput out = new BytesStreamOutput()) { @@ -348,7 +326,7 @@ public void addMetric(String fieldName) { } private void add(String fieldName, BytesReference encoded) throws IOException { - final DimensionDataHolder dimension = new DimensionDataHolder(fieldName, encoded); + final DimensionDataHolder dimension = new DimensionDataHolder(new BytesRef(fieldName), encoded.toBytesRef()); if (dimensions.contains(dimension)) { throw new IllegalArgumentException("Dimension field [" + fieldName + "] cannot be a multi-valued field."); } From 8c2b8aa73bed5bfbbcd63514ca5221c61c7eabce Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 30 Oct 2023 17:03:39 +0100 Subject: [PATCH 024/125] fix: update some tsid values in tests Getting documents by id still doesn't work because we need to use the tsid to correctly create matching documents ids. --- .../test/aggregations/time_series.yml | 12 +-- .../test/data_stream/150_tsdb.yml | 6 +- .../test/tsdb/140_routing_path.yml | 12 +-- .../test/tsdb/25_id_generation.yml | 84 ++++++++++++++----- .../rest-api-spec/test/tsdb/30_snapshot.yml | 2 +- .../rest-api-spec/test/tsdb/40_search.yml | 14 ++-- .../rest-api-spec/test/tsdb/50_alias.yml | 10 +-- .../test/tsdb/60_add_dimensions.yml | 10 +-- .../test/tsdb/70_dimension_types.yml | 44 +++++----- .../test/tsdb/80_index_resize.yml | 4 +- .../support/TimeSeriesDimensionsLimitIT.java | 8 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 34 ++++++-- .../mapper/TimeSeriesIdFieldMapperTests.java | 22 ++--- 13 files changed, 166 insertions(+), 96 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index d0dcd7f0d2fab..ac5725534f06f 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -76,7 +76,7 @@ setup: - match: { hits.total.value: 1 } - length: { aggregations: 1 } - - match: { aggregations.ts.buckets.0.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACv9hou4__k9oZfeombF5dLPwkqUA" } } + - match: { aggregations.ts.buckets.0.key: { _tsid: "QxzdeAQNbMXbXqwEXmfHBQAAAAAAAAAAAAAAAAAAAACv9hou4__k9oZfeombF5dLPwkqUA" } } - match: { aggregations.ts.buckets.0.doc_count: 1 } --- @@ -126,7 +126,7 @@ setup: size: 1 - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACKf_6WZzMy4zeMjdIBoQC3xHGkjg" } } + - match: { aggregations.ts.buckets.0.key: { _tsid: "QxzdeAQNbMXbXqwEXmfHBQAAAAAAAAAAAAAAAAAAAACKf_6WZzMy4zeMjdIBoQC3xHGkjg" } } - do: search: @@ -144,9 +144,9 @@ setup: size: 3 - length: { aggregations.ts.buckets: 3 } - - match: { aggregations.ts.buckets.0.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACKf_6WZzMy4zeMjdIBoQC3xHGkjg" } } - - match: { aggregations.ts.buckets.1.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAACv9hou4__k9oZfeombF5dLPwkqUA" } } - - match: { aggregations.ts.buckets.2.key: { _tsid: "vD4dAfOH-KDNoywVO13gvAAAAAAAAAAAAAAAAAAAAAC5AZB2-M4K_VYiHrKDjUNX3zHVkw" } } + - match: { aggregations.ts.buckets.0.key: { _tsid: "QxzdeAQNbMXbXqwEXmfHBQAAAAAAAAAAAAAAAAAAAACKf_6WZzMy4zeMjdIBoQC3xHGkjg" } } + - match: { aggregations.ts.buckets.1.key: { _tsid: "QxzdeAQNbMXbXqwEXmfHBQAAAAAAAAAAAAAAAAAAAACv9hou4__k9oZfeombF5dLPwkqUA" } } + - match: { aggregations.ts.buckets.2.key: { _tsid: "QxzdeAQNbMXbXqwEXmfHBQAAAAAAAAAAAAAAAAAAAAC5AZB2-M4K_VYiHrKDjUNX3zHVkw" } } --- "Score test filter some": @@ -341,5 +341,5 @@ setup: - match: { hits.total.value: 2 } - length: { aggregations: 1 } - - match: { aggregations.ts.buckets.0.key: { "key": "10" } } + - match: { aggregations.ts.buckets.0.key: { "_tsid": "QxzdeAQNbMXbXqwEXmfHBQAAAAAAAAAAAAAAAAAAAAC6KPQ_9D8hCF3DOnGNi7WMdDeQmw" } } - match: { aggregations.ts.buckets.0.doc_count: 1 } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 10af5906eee80..a0e67c61d076e 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -147,7 +147,7 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" }]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" }]} --- "aggregate the tsid": @@ -168,9 +168,9 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 3b2c0d37a3f26..9b7052fd80810 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -78,22 +78,22 @@ missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6Oix0Ruaua4Wo_hP7e-sEqo5ihHDIG1DaFBdVI_fYOQvJgKOvg" } + - match: { aggregations.tsids.buckets.0.key._tsid: "3LQFBfjW0o1rcoR2NF5cz-ix0Ruaua4Wo_hP7e-sEqo5ihHDIG1DaFBdVI_fYOQvJgKOvg" } - match: { aggregations.tsids.buckets.0.key.tag: null } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6Oix0Ruaua4Wo_hP7e-sEqp12oDh9NI69d0Kk5TQ6CAdewYP5A" } + - match: { aggregations.tsids.buckets.1.key._tsid: "3LQFBfjW0o1rcoR2NF5cz-ix0Ruaua4Wo_hP7e-sEqp12oDh9NI69d0Kk5TQ6CAdewYP5A" } - match: { aggregations.tsids.buckets.1.key.tag: null } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.15, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqqaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } + - match: { aggregations.tsids.buckets.2.key._tsid: "M4NGZuPu-7ista-OslulS-ix0Ruaua4Wo_hP7e-sEqqaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } - match: { aggregations.tsids.buckets.2.key.tag: null } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqrcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } + - match: { aggregations.tsids.buckets.3.key._tsid: "M4NGZuPu-7ista-OslulS-ix0Ruaua4Wo_hP7e-sEqrcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } - match: { aggregations.tsids.buckets.3.key.tag: null } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 7.30, error: 0.01 }} @@ -209,12 +209,12 @@ multi-value routing path field: - match: {hits.total.value: 4} - length: {aggregations.tsids.buckets: 2} - - match: {aggregations.tsids.buckets.0.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqqaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } + - match: {aggregations.tsids.buckets.0.key._tsid: "M4NGZuPu-7ista-OslulS-ix0Ruaua4Wo_hP7e-sEqqaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } - match: {aggregations.tsids.buckets.0.key.tag: null } - match: {aggregations.tsids.buckets.0.doc_count: 2 } - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "qSFbO7EapWUonTzXAt9gquix0Ruaua4Wo_hP7e-sEqrcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } + - match: { aggregations.tsids.buckets.1.key._tsid: "M4NGZuPu-7ista-OslulS-ix0Ruaua4Wo_hP7e-sEqrcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } - match: { aggregations.tsids.buckets.1.key.tag: null } - match: {aggregations.tsids.buckets.1.doc_count: 2 } - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 81d3417820f1c..b0babe1302733 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -76,7 +76,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs4NdV58ePSPIAAABeRnS7fM} + - match: {items.0.index._id: cZZNs2vcxybH5Kk3AAABeRnS7fM} - do: bulk: @@ -85,7 +85,53 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs4NdV58ePSPIAAABeRnS7fM} + - match: {items.0.index._id: cZZNs2vcxybH5Kk3AAABeRnS7fM} + + - do: + search: + index: test + body: + query: + match_all: {} + sort: ["@timestamp"] + + - match: {hits.total.value: 9} + + - match: { hits.hits.0._id: cn4exSPK93Q9eJj8AAABeRnRFAY } + - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.1._id: cZZNs-xII2fZweptAAABeRnRGTM } + - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:04.467Z } + - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.2._id: cn4exSPK93Q9eJj8AAABeRnRYiY } + - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:23.142Z } + - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.3._id: cZZNs-xII2fZweptAAABeRnRZ1M } + - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:24.467Z } + - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.4._id: cZZNs-xII2fZweptAAABeRnRtXM } + - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:44.467Z } + - match: { hits.hits.4._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.5._id: cn4exSPK93Q9eJj8AAABeRnR11Y } + - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:50:53.142Z } + - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.6._id: cn4exSPK93Q9eJj8AAABeRnR_mY } + - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:03.142Z } + - match: { hits.hits.6._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + + - match: { hits.hits.7._id: cZZNs-xII2fZweptAAABeRnSA5M } + - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:51:04.467Z } + - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + + - match: { hits.hits.8._id: cZZNs-xII2fZweptAAABeRnS7fM } + - match: { hits.hits.8._source.@timestamp: 2021-04-28T18:52:04.467Z } + - match: { hits.hits.8._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } --- index a new document on top of an old one: @@ -124,7 +170,7 @@ index a new document on top of an old one: network: tx: 111434595272 rx: 430605511 - - match: {_id: cn4exTOUtxytuLkQAAABeRnR_mY} + - match: {_id: cn4exVlxKdyGtXayAAABeRnR_mY} - do: search: @@ -135,11 +181,11 @@ index a new document on top of an old one: max_tx: max: field: k8s.pod.network.tx - max_rx: + min_rx: min: field: k8s.pod.network.rx - match: {aggregations.max_tx.value: 1.11434595272E11} - - match: {aggregations.max_rx.value: 4.30605511E8} + - match: {aggregations.min_rx.value: 4.30605511E8} --- index a new document on top of an old one over bulk: @@ -169,7 +215,7 @@ index a new document on top of an old one over bulk: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: {items.0.index._id: cn4exTOUtxytuLkQAAABeRnR_mY} + - match: {items.0.index._id: cn4exVlxKdyGtXayAAABeRnR_mY} - do: search: @@ -193,7 +239,7 @@ create operation on top of old document fails: reason: id generation changed in 8.2 - do: - catch: "/\\[cn4exTOUtxytuLkQAAABeRnR_mY\\]\\[\\{.+\\}\\@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" + catch: "/\\[cn4exVlxKdyGtXayAAABeRnR_mY\\]\\[\\{.+\\}\\@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" index: refresh: true index: test @@ -222,7 +268,7 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[cn4exTOUtxytuLkQAAABeRnR_mY][{_tsid=jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[cn4exVlxKdyGtXayAAABeRnR_mY][{_tsid=KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: @@ -238,12 +284,12 @@ ids query: - field: k8s.pod.network.tx query: ids: - values: ["cn4exTOUtxytuLkQAAABeRnR_mY", "cZZNs4NdV58ePSPIAAABeRnSA5M"] + values: ["cn4exSPK93Q9eJj8AAABeRnR_mY", "cZZNs-xII2fZweptAAABeRnSA5M"] sort: ["@timestamp"] - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "cn4exafToJ7Wv1jaAAABeRnR_mY"} + - match: {hits.hits.0._id: "cn4exSPK93Q9eJj8AAABeRnR_mY"} - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434595272]} - - match: {hits.hits.1._id: "cZZNs8_QMMcY0B43AAABeRnSA5M"} + - match: {hits.hits.1._id: "cZZNs-xII2fZweptAAABeRnSA5M"} - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [2012916202]} --- @@ -255,9 +301,9 @@ get: - do: get: index: test - id: cZZNs4NdV58ePSPIAAABeRnSA5M + id: cZZNs-xII2fZweptAAABeRnSA5M - match: {_index: test} - - match: {_id: cZZNs4NdV58ePSPIAAABeRnSA5M} + - match: {_id: cZZNs-xII2fZweptAAABeRnSA5M} - match: _source: "@timestamp": "2021-04-28T18:51:04.467Z" @@ -305,7 +351,7 @@ delete: - do: delete: index: test - id: cZZNs4NdV58ePSPIAAABeRnSA5M + id: cn4exSPK93Q9eJj8AAABeRnRFAY - match: {result: deleted} --- @@ -345,12 +391,12 @@ delete over _bulk: mget: index: test body: - ids: [ cn4exTOUtxytuLkQAAABeRnR_mY, cZZNs4NdV58ePSPIAAABeRnSA5M ] + ids: [ cn4exSPK93Q9eJj8AAABeRnR_mY, cZZNs-xII2fZweptAAABeRnSA5M ] - match: { docs.0._index: "test" } - - match: { docs.0._id: "cn4exTOUtxytuLkQAAABeRnR_mY" } + - match: { docs.0._id: "cn4exSPK93Q9eJj8AAABeRnR_mY" } - match: { docs.0.found: true } - match: { docs.1._index: "test" } - - match: { docs.1._id: "cZZNs4NdV58ePSPIAAABeRnSA5M" } + - match: { docs.1._id: "cZZNs-xII2fZweptAAABeRnSA5M" } - match: { docs.1.found: true } - do: @@ -408,7 +454,7 @@ routing_path matches deep object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"bar": {"baz": {"uid": "uid1"}}}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: OcEOGaxBa0saxogMAAABeRnRGTM} + - match: {items.0.index._id: OcEOGWojqLcS93iiAAABeRnRGTM} --- routing_path matches object: @@ -449,4 +495,4 @@ routing_path matches object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"uid": "uid1"}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: 8bgiqUyQKH6n8noAAAABeRnRGTM} + - match: {items.0.index._id: 8bgiqVXBrZn1EpjcAAABeRnRGTM} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 18c24b157d937..d150a73ed7e01 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -148,5 +148,5 @@ teardown: - match: {hits.total.value: 1} - match: {hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507} - - match: {hits.hits.0.fields._tsid: [{ _tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" }]} + - match: {hits.hits.0.fields._tsid: [{ _tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" }]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index da4ef36ae78f5..b6678582b08ab 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -185,7 +185,7 @@ fetch a tag: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}]} --- aggregate a dimension: @@ -282,9 +282,9 @@ aggregate a tag: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -319,7 +319,7 @@ sort by tsid: sort: [ "_tsid", "@timestamp" ] - match: {hits.total.value: 8} - - match: {hits.hits.0.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}, 1619635803142]} - - match: {hits.hits.1.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}, 1619635823142]} - - match: {hits.hits.4.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}, 1619635804467]} - - match: {hits.hits.7.sort: [{ _tsid : "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}, 1619635864467]} + - match: {hits.hits.0.sort: [{ _tsid : "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}, 1619635803142]} + - match: {hits.hits.1.sort: [{ _tsid : "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}, 1619635823142]} + - match: {hits.hits.4.sort: [{ _tsid : "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}, 1619635804467]} + - match: {hits.hits.7.sort: [{ _tsid : "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}, 1619635864467]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index 2325f3a383e25..00005d3e1881a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -85,9 +85,9 @@ search an alias: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -129,10 +129,10 @@ index into alias: _key: asc - match: {hits.total.value: 12} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - match: {aggregations.tsids.buckets.2.key: {_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAC-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"}} + - match: {aggregations.tsids.buckets.2.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAC-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"}} - match: {aggregations.tsids.buckets.2.doc_count: 4} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index ea37e9417a3ef..2bf6c00874ddc 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -54,7 +54,7 @@ add dimensions with put_mapping: - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "95YnAT0C6QOgFs_zuty0ZAAAAAAAAAAAAAAAAAAAAAAGlT_5e6_NYGOZWULpmMG9IAlZlA" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "27hcKTYkXCOfYiuRWRTWygAAAAAAAAAAAAAAAAAAAAAGlT_5e6_NYGOZWULpmMG9IAlZlA" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -102,7 +102,7 @@ add dimensions to no dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "95YnAT0C6QOgFs_zuty0ZAAAAAAAAAAAAAAAAAAAAAAGlT_5e6_NYGOZWULpmMG9IAlZlA" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "27hcKTYkXCOfYiuRWRTWygAAAAAAAAAAAAAAAAAAAAAGlT_5e6_NYGOZWULpmMG9IAlZlA" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -150,7 +150,7 @@ add dimensions to no dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "95YnAT0C6QOgFs_zuty0ZAAAAAAAAAAAAAAAAAAAAAAGlT_5e6_NYGOZWULpmMG9IAlZlA" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "27hcKTYkXCOfYiuRWRTWygAAAAAAAAAAAAAAAAAAAAAGlT_5e6_NYGOZWULpmMG9IAlZlA" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -202,7 +202,7 @@ add dimensions to some dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "xhgWVhgKGfmIwHt-7D2B_gAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y"} ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "-CXfM53tOdTdJn8GSJ1F1QAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y"} ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -253,5 +253,5 @@ add dimensions to some dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "xhgWVhgKGfmIwHt-7D2B_gAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y" } ] } + - match: {hits.hits.0.fields._tsid: [ { _tsid: "-CXfM53tOdTdJn8GSJ1F1QAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y" } ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 8c7e516b40588..e1c14a28e7d27 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -71,10 +71,10 @@ keyword dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAA5ihHDIG1DaFBdVI_fYOQvJgKOvg"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "3LQFBfjW0o1rcoR2NF5czwAAAAAAAAAAAAAAAAAAAAA5ihHDIG1DaFBdVI_fYOQvJgKOvg"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAB12oDh9NI69d0Kk5TQ6CAdewYP5A"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "3LQFBfjW0o1rcoR2NF5czwAAAAAAAAAAAAAAAAAAAAB12oDh9NI69d0Kk5TQ6CAdewYP5A"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -150,7 +150,7 @@ flattened dimension: - match: { hits.total.value: 8} - length: { aggregations.tsids.buckets: 4} - - match: { aggregations.tsids.buckets.0.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAAAoaZ29eRDHR3kQx0fjCuTKddqA4R9ytcYdqdbcoJ8VBuvqFtQ" } + - match: { aggregations.tsids.buckets.0.key._tsid: "ItgQI_4ahoh-OTREF92WDQAAAAAAAAAAAAAAAAAAAAAoaZ29eRDHR3kQx0fjCuTKddqA4R9ytcYdqdbcoJ8VBuvqFtQ" } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.0.key.deployment\.version\.minor: null } @@ -158,7 +158,7 @@ flattened dimension: - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAACrnf7qeRDHR3kQx0eLDvTuOYoRw4EeUIKK1KGFPmxqCWQyQJE" } + - match: { aggregations.tsids.buckets.1.key._tsid: "ItgQI_4ahoh-OTREF92WDQAAAAAAAAAAAAAAAAAAAACrnf7qeRDHR3kQx0eLDvTuOYoRw4EeUIKK1KGFPmxqCWQyQJE" } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.1.key.deployment\.version\.minor: null } @@ -166,7 +166,7 @@ flattened dimension: - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAADP1l1UeRDHR3kQx0fjCuTKddqA4ai7CiSuCxA-PhBdLYOXkVY" } + - match: { aggregations.tsids.buckets.2.key._tsid: "ItgQI_4ahoh-OTREF92WDQAAAAAAAAAAAAAAAAAAAADP1l1UeRDHR3kQx0fjCuTKddqA4ai7CiSuCxA-PhBdLYOXkVY" } - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.2.key.deployment\.version\.minor: null } @@ -174,7 +174,7 @@ flattened dimension: - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "P5PJzRH8-Gbp6mqjByZOmgAAAAAAAAAAAAAAAAAAAADgKelzeRDHR3kQx0eLDvTuOYoRw6Wapg_P07YIhdV3NFJDjjE" } + - match: { aggregations.tsids.buckets.3.key._tsid: "ItgQI_4ahoh-OTREF92WDQAAAAAAAAAAAAAAAAAAAADgKelzeRDHR3kQx0eLDvTuOYoRw6Wapg_P07YIhdV3NFJDjjE" } - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.3.key.deployment\.version\.minor: null } @@ -254,11 +254,11 @@ flattened empty dimension: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 2 } - - match: { aggregations.tsids.buckets.0.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAA5ihHDIG1DaFBdVI_fYOQvJgKOvg" } + - match: { aggregations.tsids.buckets.0.key._tsid: "3LQFBfjW0o1rcoR2NF5czwAAAAAAAAAAAAAAAAAAAAA5ihHDIG1DaFBdVI_fYOQvJgKOvg" } - match: { aggregations.tsids.buckets.0.doc_count: 4 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "Dh_r6QMn0c6jJhFGYOwd6AAAAAAAAAAAAAAAAAAAAAB12oDh9NI69d0Kk5TQ6CAdewYP5A" } + - match: { aggregations.tsids.buckets.1.key._tsid: "3LQFBfjW0o1rcoR2NF5czwAAAAAAAAAAAAAAAAAAAAB12oDh9NI69d0Kk5TQ6CAdewYP5A" } - match: { aggregations.tsids.buckets.1.doc_count: 4 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.22, error: 0.01 }} @@ -334,42 +334,42 @@ flattened field missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 6 } - - match: { aggregations.tsids.buckets.0.key._tsid: "s0XCB0MdYlNxY9PEJqOWhgAAAAAAAAAAAAAAAAAAAADxTXebeRDHRzmKEcOUW9u_kzgC7pSzoi1utVcm" } + - match: { aggregations.tsids.buckets.0.key._tsid: "8HNIqk_KxgNH6XWO9gDyMwAAAAAAAAAAAAAAAAAAAADxTXebeRDHRzmKEcOUW9u_kzgC7pSzoi1utVcm" } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.0.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "s0XCB0MdYlNxY9PEJqOWhgAAAAAAAAAAAAAAAAAAAADxTXebeRDHR3XagOEjGQJfwIB2Q5kLDL76leH4" } + - match: { aggregations.tsids.buckets.1.key._tsid: "8HNIqk_KxgNH6XWO9gDyMwAAAAAAAAAAAAAAAAAAAADxTXebeRDHR3XagOEjGQJfwIB2Q5kLDL76leH4" } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.1.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.1.doc_count: 1 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXebKGmdvXkQx0d12oDh16FtfvP_ObklCa7gXDcmdA" } + - match: { aggregations.tsids.buckets.2.key._tsid: "XXEWRB81VefgRtsBbneB7wAAAAAAAAAAAAAAAAAAAADxTXebKGmdvXkQx0d12oDh16FtfvP_ObklCa7gXDcmdA" } - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.2.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.2.doc_count: 1 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXebq53-6nkQx0c5ihHDU3PuzTI1mM98gQqqihDoZA" } + - match: { aggregations.tsids.buckets.3.key._tsid: "XXEWRB81VefgRtsBbneB7wAAAAAAAAAAAAAAAAAAAADxTXebq53-6nkQx0c5ihHDU3PuzTI1mM98gQqqihDoZA" } - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.3.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.4.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXebz9ZdVHkQx0d12oDhi1SccpVnRAhpjCrWhqFjfQ" } + - match: { aggregations.tsids.buckets.4.key._tsid: "XXEWRB81VefgRtsBbneB7wAAAAAAAAAAAAAAAAAAAADxTXebz9ZdVHkQx0d12oDhi1SccpVnRAhpjCrWhqFjfQ" } - match: { aggregations.tsids.buckets.4.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.4.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.4.key.deployment\.version\.major: null } - match: { aggregations.tsids.buckets.4.doc_count: 2 } - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.5.key._tsid: "WPttUWoKR4uM6b9jH-_EXgAAAAAAAAAAAAAAAAAAAADxTXeb4Cnpc3kQx0c5ihHDqaS5hHgLAm0E3XkHG7fdkQ" } + - match: { aggregations.tsids.buckets.5.key._tsid: "XXEWRB81VefgRtsBbneB7wAAAAAAAAAAAAAAAAAAAADxTXeb4Cnpc3kQx0c5ihHDqaS5hHgLAm0E3XkHG7fdkQ" } - match: { aggregations.tsids.buckets.5.key.deployment\.build\.tag: null } - match: { aggregations.tsids.buckets.5.key.deployment\.build\.branch: null } - match: { aggregations.tsids.buckets.5.key.deployment\.version\.major: null } @@ -452,19 +452,19 @@ flattened field misspelled routing path field: - match: { hits.total.value: 6 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAAAoaZ29egRdyNeHXPSPghDVzguaRg" } + - match: { aggregations.tsids.buckets.0.key._tsid: "GzT4lnbKgCD4uvDfky7fKwAAAAAAAAAAAAAAAAAAAAAoaZ29egRdyNeHXPSPghDVzguaRg" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAACrnf7qs9-VXFZ6jjZCbl_iiXSs7Q" } + - match: { aggregations.tsids.buckets.1.key._tsid: "GzT4lnbKgCD4uvDfky7fKwAAAAAAAAAAAAAAAAAAAACrnf7qs9-VXFZ6jjZCbl_iiXSs7Q" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAADP1l1UlmlXEQVNXrHpUvpn7by0jA" } + - match: { aggregations.tsids.buckets.2.key._tsid: "GzT4lnbKgCD4uvDfky7fKwAAAAAAAAAAAAAAAAAAAADP1l1UlmlXEQVNXrHpUvpn7by0jA" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "Vpe7uQk6zUIFfFcpASZ8YwAAAAAAAAAAAAAAAAAAAADgKelz9WSJqzeYh7aza_7yxDXMZA" } + - match: { aggregations.tsids.buckets.3.key._tsid: "GzT4lnbKgCD4uvDfky7fKwAAAAAAAAAAAAAAAAAAAADgKelz9WSJqzeYh7aza_7yxDXMZA" } - match: { aggregations.tsids.buckets.3.doc_count: 1 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} @@ -536,10 +536,10 @@ long dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "XNiiZwxkczx2lrb8CBwniwAAAAAAAAAAAAAAAAAAAABDgKYd73outpXn7LJV-gQfvlrec7NyMho"}} + - match: {aggregations.tsids.buckets.0.key: {_tsid: "xq55J0GFz9YiFjjGg8TYRwAAAAAAAAAAAAAAAAAAAABDgKYd73outpXn7LJV-gQfvlrec7NyMho"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "XNiiZwxkczx2lrb8CBwniwAAAAAAAAAAAAAAAAAAAACtm41v73outmoSTcFmfQBYjOjMaOWM5zs"}} + - match: {aggregations.tsids.buckets.1.key: {_tsid: "xq55J0GFz9YiFjjGg8TYRwAAAAAAAAAAAAAAAAAAAACtm41v73outmoSTcFmfQBYjOjMaOWM5zs"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -611,10 +611,10 @@ ip dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: { _tsid: "Wlq2NkhwVgBD7x8-uwLcHQAAAAAAAAAAAAAAAAAAAADYzb-v73outosB-eDb_nzTrIVsJFVQR2c"}} + - match: {aggregations.tsids.buckets.0.key: { _tsid: "tWt3sTyhkvJ3619KbQfqMAAAAAAAAAAAAAAAAAAAAADYzb-v73outosB-eDb_nzTrIVsJFVQR2c"}} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: { _tsid: "Wlq2NkhwVgBD7x8-uwLcHQAAAAAAAAAAAAAAAAAAAADaaOFK73outkqpG8R65Gm4VUhpyuc11zw"}} + - match: {aggregations.tsids.buckets.1.key: { _tsid: "tWt3sTyhkvJ3619KbQfqMAAAAAAAAAAAAAAAAAAAAADaaOFK73outkqpG8R65Gm4VUhpyuc11zw"}} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index a4d4d594237e9..0ed6573c9ea19 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -124,7 +124,7 @@ shrink: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}]} --- clone: @@ -148,4 +148,4 @@ clone: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "jsPn2w3PUBDHCdUJaEUx2gAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}]} + - match: {hits.hits.0.fields._tsid: [{_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}]} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java index 30445b5220b20..3fb85e795b1b9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java @@ -43,7 +43,7 @@ public void testDimensionFieldNameLimit() throws IOException { () -> List.of("routing_field"), dimensionFieldLimit ); - final IndexResponse response = client().prepareIndex("test") + final DocWriteResponse response = client().prepareIndex("test") .setSource( "routing_field", randomAlphaOfLength(10), @@ -67,10 +67,10 @@ public void testDimensionFieldValueLimit() throws IOException { dimensionFieldLimit ); long startTime = Instant.now().toEpochMilli(); - final IndexResponse response1 = client().prepareIndex("test") + final DocWriteResponse response1 = client().prepareIndex("test") .setSource("field", randomAlphaOfLength(1024), "gauge", randomIntBetween(10, 20), "@timestamp", startTime) .get(); - final IndexResponse response2 = client().prepareIndex("test") + final DocWriteResponse response2 = client().prepareIndex("test") .setSource("field", randomAlphaOfLength(1025), "gauge", randomIntBetween(10, 20), "@timestamp", startTime + 1) .get(); assertEquals(RestStatus.CREATED.getStatus(), response1.status().getStatus()); @@ -156,7 +156,7 @@ public void testTotalDimensionFieldsSizeLuceneLimitPlusOne() throws IOException for (int i = 0; i < dimensionFieldLimit; i++) { source.put(dimensionFieldNames.get(i), randomAlphaOfLength(1024)); } - final IndexResponse response = client().prepareIndex("test").setSource(source).get(); + final DocWriteResponse response = client().prepareIndex("test").setSource(source).get(); assertEquals(RestStatus.CREATED.getStatus(), response.status().getStatus()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index ee1026fe38b28..1dd3b627b979f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -11,13 +11,16 @@ import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.StringHelper; import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.hash.Murmur3Hasher; +import org.elasticsearch.common.hash.MurmurHash3; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.fielddata.FieldData; @@ -237,36 +240,57 @@ public BytesReference build() throws IOException { public BytesReference similarityHash() throws IOException { // NOTE: hash all dimension field names int numberOfDimensions = Math.min(MAX_DIMENSIONS, dimensions.size()); + int tsidHashIndex = 0; + byte[] tsidHash = new byte[16 + 16 + 16 + 4 * numberOfDimensions]; + tsidHasher.reset(); for (final DimensionDataHolder dimension : dimensions) { tsidHasher.update(dimension.name.bytes); } + tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); // NOTE: hash all metric field names + tsidHasher.reset(); for (final String metric : metrics) { tsidHasher.update(metric.getBytes(StandardCharsets.UTF_8)); } + tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); // NOTE: concatenate all dimension value hashes up to a certain number of dimensions - int dimensionCounter = 0; + int tsidHashStartIndex = tsidHashIndex; for (final DimensionDataHolder dimension : dimensions) { - if (dimensionCounter > numberOfDimensions) break; - tsidHasher.update(dimension.value.bytes); - dimensionCounter++; + if ((tsidHashIndex - tsidHashStartIndex) >= 4 * numberOfDimensions) break; + ByteUtils.writeIntLE( + StringHelper.murmurhash3_x86_32(dimension.value.bytes, dimension.value.offset, dimension.value.length, 0), + tsidHash, + tsidHashIndex + ); + tsidHashIndex += 4; } // NOTE: hash all dimension field allValues + tsidHasher.reset(); for (final DimensionDataHolder dimension : dimensions) { tsidHasher.update(dimension.value.bytes); } + tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); + assert tsidHashIndex == tsidHash.length; try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(TSID_HASH_SENTINEL); - out.writeBytesRef(new BytesRef(tsidHasher.digest())); + out.writeBytesRef(new BytesRef(tsidHash, 0, tsidHash.length)); return out.bytes(); } } + private int writeHash128(final MurmurHash3.Hash128 hash128, byte[] buffer, int tsidHashIndex) { + ByteUtils.writeLongLE(hash128.h1, buffer, tsidHashIndex); + tsidHashIndex += 8; + ByteUtils.writeLongLE(hash128.h2, buffer, tsidHashIndex); + tsidHashIndex += 8; + return tsidHashIndex; + } + @Override public void addKeywordDimension(String fieldName, BytesRef utf8Value) { try (BytesStreamOutput out = new BytesStreamOutput()) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index f2fa94c759af0..483b60754cef2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -79,7 +79,7 @@ public void testEnabledInTimeSeriesMode() throws Exception { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100).field("c", 500)); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "LhvtFuoRi5Ot1FKbAade5gAAAAAAAAAAAAAAAAAAAADuDjjxPwT2Ol0UAzaqxX0Rpx75AkZHRdY") + matchesMap().entry("_tsid", "30CO74tuRyatuMXEvNJvbgAAAAAAAAAAAAAAAAAAAADuDjjxPwT2Ol0UAzaqxX0Rpx75AkZHRdY") ); } @@ -126,7 +126,7 @@ public void testStrings() throws IOException { ); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "-uh8SqJqlbcmNjsxQifzOQAAAAAAAAAAAAAAAAAAAACv9houtA-WtALkiErAWD5o8PqlkO2CPtw") + matchesMap().entry("_tsid", "S3oqtElkKwgW-mWA1nr9gAAAAAAAAAAAAAAAAAAAAACv9houtA-WtALkiErAWD5o8PqlkO2CPtw") ); } @@ -142,7 +142,7 @@ public void testUnicodeKeys() throws IOException { Map tsid = TimeSeriesIdFieldMapper.decodeTsid( new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes) ); - assertMap(tsid, matchesMap().entry("_tsid", "Jp7d1TeAbnqXtvYiILWo7gAAAAAAAAAAAAAAAAAAAACejAXjDKlQ6oAVV6S1YhlDnYAgGZ4kqeg")); + assertMap(tsid, matchesMap().entry("_tsid", "A-nLhW-M69syDAiaaHLtCgAAAAAAAAAAAAAAAAAAAACejAXjDKlQ6oAVV6S1YhlDnYAgGZ4kqeg")); } public void testKeywordTooLong() throws IOException { @@ -153,7 +153,7 @@ public void testKeywordTooLong() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "iXhZ9mVVVYVaiQ5RSDq15gAAAAAAAAAAAAAAAAAAAAC9ZDUoCO2eynkD-lZbEGc5huKIYw") + matchesMap().entry("_tsid", "FlVJQe85E3Mv4Ekz_gxSawAAAAAAAAAAAAAAAAAAAAC9ZDUoCO2eynkD-lZbEGc5huKIYw") ); } @@ -166,7 +166,7 @@ public void testKeywordTooLongUtf8() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "iXhZ9mVVVYVaiQ5RSDq15gAAAAAAAAAAAAAAAAAAAABM43f6n6D4bFDIodLIvFw5j_1Vew") + matchesMap().entry("_tsid", "FlVJQe85E3Mv4Ekz_gxSawAAAAAAAAAAAAAAAAAAAABM43f6n6D4bFDIodLIvFw5j_1Vew") ); } @@ -207,7 +207,7 @@ public void testLong() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJy91wa2A3yf66JQYgw9_3GPhrunFA") + matchesMap().entry("_tsid", "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJy91wa2A3yf66JQYgw9_3GPhrunFA") ); } @@ -261,7 +261,7 @@ public void testInteger() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJy9XyEVAdPjbPnyFfNkppmgRrzARX") + matchesMap().entry("_tsid", "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJy9XyEVAdPjbPnyFfNkppmgRrzARX") ); } @@ -319,7 +319,7 @@ public void testShort() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJyyYAWY_JC1ss-P30niOYXA2x9kLq") + matchesMap().entry("_tsid", "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJyyYAWY_JC1ss-P30niOYXA2x9kLq") ); } @@ -377,7 +377,7 @@ public void testByte() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAACtm41vSrpJyygrqocRYcbJqQJZlIyTzrY9S9Nt") + matchesMap().entry("_tsid", "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJyygrqocRYcbJqQJZlIyTzrY9S9Nt") ); } @@ -435,7 +435,7 @@ public void testIp() throws IOException { }); assertMap( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "y2oH44MiIyaUP_cSq588QQAAAAAAAAAAAAAAAAAAAAALrOlWSrpJy7PzTuwgaqmbsw6Oz00Ir_8Cw9Xm") + matchesMap().entry("_tsid", "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAAALrOlWSrpJy7PzTuwgaqmbsw6Oz00Ir_8Cw9Xm") ); } @@ -473,7 +473,7 @@ public void testVeryLarge() throws IOException { TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), matchesMap().entry( "_tsid", - "VuMQB6AxmJbsu4Fzii310QAAAAAAAAAAAAAAAAAAAACv9houWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk8VBv2rLPN9GYxF8e8uXojI" + "QZ1K9tk9UIcpA826eU6H-QAAAAAAAAAAAAAAAAAAAACv9houWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk8VBv2rLPN9GYxF8e8uXojI" ) ); } From acc0d14167be057f29fc459b28890c223017b806 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 31 Oct 2023 16:05:15 +0100 Subject: [PATCH 025/125] fix: document id generation --- .../timeseries/InternalTimeSeriesTests.java | 2 +- .../timeseries/TimeSeriesAggregatorTests.java | 2 +- .../test/tsdb/25_id_generation.yml | 22 +++++++++---------- .../elasticsearch/index/IndexVersions.java | 1 + .../index/mapper/TimeSeriesIdFieldMapper.java | 16 ++++++++++---- .../elasticsearch/search/DocValueFormat.java | 2 +- .../index/mapper/IdLoaderTests.java | 4 ++-- .../search/DocValueFormatTests.java | 2 +- .../rate/TimeSeriesRateAggregatorTests.java | 2 +- .../aggregations/GeoLineAggregatorTests.java | 2 +- 10 files changed, 32 insertions(+), 23 deletions(-) diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java index a9e22010fb164..2f3f195754f2a 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java @@ -53,7 +53,7 @@ private List randomBuckets(boolean keyed, InternalAggregations a builder.addKeywordDimension(entry.getKey(), (String) entry.getValue()); } try { - var key = builder.build().toBytesRef(); + var key = builder.withoutHash().toBytesRef(); bucketList.add(new InternalBucket(key, docCount, aggregations, keyed)); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index 49852331c2b54..cf26c21f261c2 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -101,7 +101,7 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens fields.add(new DoubleDocValuesField(metrics[i].toString(), (double) metrics[i + 1])); } } - fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.build().toBytesRef())); + fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.withoutHash().toBytesRef())); // TODO: Handle metrics iw.addDocument(fields); } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index b0babe1302733..76cf549854a01 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -76,7 +76,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs2vcxybH5Kk3AAABeRnS7fM} + - match: {items.0.index._id: cZZNs-xII2fZweptAAABeRnS7fM} - do: bulk: @@ -85,7 +85,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs2vcxybH5Kk3AAABeRnS7fM} + - match: {items.0.index._id: cZZNs-xII2fZweptAAABeRnS7fM} - do: search: @@ -170,7 +170,7 @@ index a new document on top of an old one: network: tx: 111434595272 rx: 430605511 - - match: {_id: cn4exVlxKdyGtXayAAABeRnR_mY} + - match: {_id: cn4exSPK93Q9eJj8AAABeRnR_mY} - do: search: @@ -215,7 +215,7 @@ index a new document on top of an old one over bulk: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: {items.0.index._id: cn4exVlxKdyGtXayAAABeRnR_mY} + - match: {items.0.index._id: cn4exSPK93Q9eJj8AAABeRnR_mY} - do: search: @@ -239,7 +239,7 @@ create operation on top of old document fails: reason: id generation changed in 8.2 - do: - catch: "/\\[cn4exVlxKdyGtXayAAABeRnR_mY\\]\\[\\{.+\\}\\@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" + catch: "/\\[cn4exSPK93Q9eJj8AAABeRnR_mY\\]\\[\\{.+\\}\\@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" index: refresh: true index: test @@ -268,7 +268,7 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[cn4exVlxKdyGtXayAAABeRnR_mY][{_tsid=KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[cn4exSPK93Q9eJj8AAABeRnR_mY][{_tsid=KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: @@ -339,7 +339,7 @@ get with routing: catch: bad_request get: index: test - id: cZZNs4NdV58ePSPIAAABeRnSA5M + id: cZZNs-xII2fZweptAAABeRnSA5M routing: routing --- @@ -403,8 +403,8 @@ delete over _bulk: bulk: index: test body: - - '{"delete": {"_id": "cn4exTOUtxytuLkQAAABeRnR_mY"}}' - - '{"delete": {"_id": "cZZNs4NdV58ePSPIAAABeRnSA5M"}}' + - '{"delete": {"_id": "cn4exSPK93Q9eJj8AAABeRnR_mY"}}' + - '{"delete": {"_id": "cZZNs-xII2fZweptAAABeRnSA5M"}}' - '{"delete": {"_id": "not found ++ not found"}}' - match: {items.0.delete.result: deleted} - match: {items.1.delete.result: deleted} @@ -454,7 +454,7 @@ routing_path matches deep object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"bar": {"baz": {"uid": "uid1"}}}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: OcEOGWojqLcS93iiAAABeRnRGTM} + - match: {items.0.index._id: OcEOGeykhIEIYuXSAAABeRnRGTM} --- routing_path matches object: @@ -495,4 +495,4 @@ routing_path matches object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"uid": "uid1"}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: 8bgiqVXBrZn1EpjcAAABeRnRGTM} + - match: {items.0.index._id: 8bgiqXPBT_mM6CX6AAABeRnRGTM} diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 6327c2ba53f54..3615cac396d28 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -87,6 +87,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion NEW_SPARSE_VECTOR = def(8_500_001, Version.LUCENE_9_7_0); public static final IndexVersion SPARSE_VECTOR_IN_FIELD_NAMES_SUPPORT = def(8_500_002, Version.LUCENE_9_7_0); public static final IndexVersion UPGRADE_LUCENE_9_8 = def(8_500_003, Version.LUCENE_9_8_0); + public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_500_004, Version.LUCENE_9_8_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 1dd3b627b979f..05b254cb20008 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -23,6 +23,8 @@ import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -136,11 +138,17 @@ public void postParse(DocumentParserContext context) throws IOException { assert fieldType().isIndexed() == false; final TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDocumentFields(); - final BytesRef timeSeriesId = timeSeriesIdBuilder.build().toBytesRef(); - context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesIdBuilder.similarityHash().toBytesRef())); + final BytesRef timeSeriesId = getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ID_HASHING) + ? timeSeriesIdBuilder.withoutHash().toBytesRef() + : timeSeriesIdBuilder.withHash().toBytesRef(); + context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); TsidExtractingIdFieldMapper.createField(context, timeSeriesIdBuilder.routingBuilder, timeSeriesId); } + private IndexVersion getIndexVersionCreated(final DocumentParserContext context) { + return context.indexSettings().getIndexVersionCreated(); + } + @Override protected String contentType() { return CONTENT_TYPE; @@ -210,7 +218,7 @@ public TimeSeriesIdBuilder(@Nullable IndexRouting.ExtractFromSource.Builder rout this.routingBuilder = routingBuilder; } - public BytesReference build() throws IOException { + public BytesReference withoutHash() throws IOException { if (dimensions.isEmpty()) { throw new IllegalArgumentException("Dimension fields are missing."); } @@ -237,7 +245,7 @@ public BytesReference build() throws IOException { * The idea is to be able to place 'similar' time series close to each other. Two time series * are considered 'similar' if they share the same dimensions (names and values). */ - public BytesReference similarityHash() throws IOException { + public BytesReference withHash() throws IOException { // NOTE: hash all dimension field names int numberOfDimensions = Math.min(MAX_DIMENSIONS, dimensions.size()); int tsidHashIndex = 0; diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index f801ed327a6a7..08625a08723cf 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -724,7 +724,7 @@ public BytesRef parseBytesRef(Object value) { } try { - return builder.build().toBytesRef(); + return builder.withoutHash().toBytesRef(); } catch (IOException e) { throw new IllegalArgumentException(e); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index e999002a61f87..8f6b5fb452de7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -234,7 +234,7 @@ private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter fields.add(new SortedSetDocValuesField(dimension.field, new BytesRef(dimension.value.toString()))); } } - BytesRef tsid = builder.build().toBytesRef(); + BytesRef tsid = builder.withoutHash().toBytesRef(); fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, tsid)); iw.addDocument(fields); } @@ -252,7 +252,7 @@ private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc return TsidExtractingIdFieldMapper.createId( false, routingBuilder, - timeSeriesIdBuilder.build().toBytesRef(), + timeSeriesIdBuilder.withoutHash().toBytesRef(), doc.timestamp, new byte[16] ); diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index ad3ba6107c8a8..eac9cee3aee82 100644 --- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -377,7 +377,7 @@ public void testParseTsid() throws IOException { timeSeriesIdBuilder.addKeywordDimension("string", randomAlphaOfLength(10)); timeSeriesIdBuilder.addLongDimension("long", randomLong()); timeSeriesIdBuilder.addUnsignedLongDimension("ulong", randomLong()); - BytesRef tsidBytes = timeSeriesIdBuilder.build().toBytesRef(); + BytesRef tsidBytes = timeSeriesIdBuilder.withoutHash().toBytesRef(); Object tsidFormat = DocValueFormat.TIME_SERIES_ID.format(tsidBytes); BytesRef tsidParse = DocValueFormat.TIME_SERIES_ID.parseBytesRef(tsidFormat); assertEquals(tsidBytes, tsidParse); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index 6af0a731aeb60..46099ebba74eb 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -162,7 +162,7 @@ private List docs(long startTimestamp, String dim, long... values) thr private static BytesReference tsid(String dim) throws IOException { TimeSeriesIdFieldMapper.TimeSeriesIdBuilder idBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); idBuilder.addKeywordDimension("dim", dim); - return idBuilder.build(); + return idBuilder.withoutHash(); } private Document doc(long timestamp, BytesReference tsid, long counterValue) { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index 64b83a29584f6..3b04c276d584e 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -803,7 +803,7 @@ private void assertGeoLine_TSDB( ArrayList fields = new ArrayList<>( Arrays.asList( new SortedDocValuesField("group_id", new BytesRef(testData.groups[g])), - new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.build().toBytesRef()) + new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.withoutHash().toBytesRef()) ) ); GeoPoint point = points.get(i); From ef33743f7e1fb3b53c73f5b98fe05e8d5936a8c1 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 3 Nov 2023 13:53:41 +0100 Subject: [PATCH 026/125] fix: use the tsid hash as a bucket key --- .../bucket/timeseries/InternalTimeSeries.java | 2 +- .../timeseries/InternalTimeSeriesTests.java | 3 +- .../InternalTimeSeriesTsidHashingTests.java | 180 +++++ .../timeseries/TimeSeriesAggregatorTests.java | 10 +- .../TimeSeriesAggregatorTsidHashingTests.java | 320 +++++++++ .../index/mapper/TimeSeriesIdFieldMapper.java | 25 +- .../elasticsearch/search/DocValueFormat.java | 2 +- .../mapper/IdLoaderTsidHashingTests.java | 275 +++++++ .../mapper/TimeSeriesIdFieldMapperTests.java | 91 ++- ...meSeriesIdFieldMapperTsidHashingTests.java | 671 ++++++++++++++++++ 10 files changed, 1538 insertions(+), 41 deletions(-) create mode 100644 modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTsidHashingTests.java create mode 100644 modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java index 44fa62fef3fa0..8604f83f6a2b6 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java @@ -81,7 +81,7 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public Map getKey() { + public Object getKey() { return TimeSeriesIdFieldMapper.decodeTsid(key); } diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java index 2f3f195754f2a..f8b1dba1d86b3 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java @@ -89,11 +89,12 @@ protected InternalTimeSeries createTestInstance(String name, Map } @Override + @SuppressWarnings("unchecked") protected void assertReduced(InternalTimeSeries reduced, List inputs) { Map, Long> keys = new HashMap<>(); for (InternalTimeSeries in : inputs) { for (InternalBucket bucket : in.getBuckets()) { - keys.compute(bucket.getKey(), (k, v) -> { + keys.compute((Map) bucket.getKey(), (k, v) -> { if (v == null) { return bucket.docCount; } else { diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTsidHashingTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTsidHashingTests.java new file mode 100644 index 0000000000000..5db7b52ea1578 --- /dev/null +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTsidHashingTests.java @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.aggregations.bucket.timeseries; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.aggregations.bucket.AggregationMultiBucketAggregationTestCase; +import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries.InternalBucket; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.MockPageCacheRecycler; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.AggregationReduceContext; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; +import org.elasticsearch.xcontent.ContextParser; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.function.Predicate; + +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.equalTo; + +public class InternalTimeSeriesTsidHashingTests extends AggregationMultiBucketAggregationTestCase { + + @Override + protected Map.Entry> getParser() { + return Map.entry(TimeSeriesAggregationBuilder.NAME, (p, c) -> ParsedTimeSeries.fromXContent(p, (String) c)); + } + + private List randomBuckets(boolean keyed, InternalAggregations aggregations) { + int numberOfBuckets = randomNumberOfBuckets(); + List bucketList = new ArrayList<>(numberOfBuckets); + List> keys = randomKeys(bucketKeys(randomIntBetween(1, 4)), numberOfBuckets); + for (int j = 0; j < numberOfBuckets; j++) { + long docCount = randomLongBetween(0, Long.MAX_VALUE / (20L * numberOfBuckets)); + var builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); + for (var entry : keys.get(j).entrySet()) { + builder.addKeywordDimension(entry.getKey(), (String) entry.getValue()); + } + try { + var key = builder.withHash().toBytesRef(); + bucketList.add(new InternalBucket(key, docCount, aggregations, keyed)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + // The interal time series' reduce method expects for each shard level response that the buckets are sorted by tsid: + bucketList.sort(Comparator.comparing(o -> o.key)); + return bucketList; + } + + private List bucketKeys(int numberOfKeys) { + return randomUnique(() -> randomAlphaOfLength(10), numberOfKeys).stream().toList(); + } + + private List> randomKeys(List bucketKeys, int numberOfBuckets) { + List> keys = new ArrayList<>(); + for (int i = 0; i < numberOfBuckets; i++) { + keys.add(randomValueOtherThanMany(keys::contains, () -> { + Map key = new TreeMap<>(); + for (String name : bucketKeys) { + key.put(name, randomAlphaOfLength(4)); + } + return key; + })); + } + return keys; + } + + @Override + protected InternalTimeSeries createTestInstance(String name, Map metadata, InternalAggregations aggregations) { + boolean keyed = randomBoolean(); + return new InternalTimeSeries(name, randomBuckets(keyed, aggregations), keyed, metadata); + } + + @Override + @SuppressWarnings("unchecked") + protected void assertReduced(InternalTimeSeries reduced, List inputs) { + Map keys = new HashMap<>(); + for (InternalTimeSeries in : inputs) { + for (InternalBucket bucket : in.getBuckets()) { + keys.compute((String) bucket.getKey(), (k, v) -> { + if (v == null) { + return bucket.docCount; + } else { + return bucket.docCount + v; + } + }); + } + } + assertThat( + reduced.getBuckets().stream().map(InternalBucket::getKey).toArray(Object[]::new), + arrayContainingInAnyOrder(keys.keySet().toArray(Object[]::new)) + ); + } + + @Override + protected Class implementationClass() { + return ParsedTimeSeries.class; + } + + @Override + protected Predicate excludePathsFromXContentInsertion() { + return s -> s.endsWith(".key"); + } + + public void testReduceSimple() { + // a simple test, to easily spot easy mistakes in the merge logic in InternalTimeSeries#reduce(...) method. + InternalTimeSeries first = new InternalTimeSeries( + "ts", + List.of( + new InternalBucket(new BytesRef("1"), 3, InternalAggregations.EMPTY, false), + new InternalBucket(new BytesRef("10"), 6, InternalAggregations.EMPTY, false), + new InternalBucket(new BytesRef("2"), 2, InternalAggregations.EMPTY, false), + new InternalBucket(new BytesRef("9"), 5, InternalAggregations.EMPTY, false) + ), + false, + Map.of() + ); + InternalTimeSeries second = new InternalTimeSeries( + "ts", + List.of( + new InternalBucket(new BytesRef("2"), 1, InternalAggregations.EMPTY, false), + new InternalBucket(new BytesRef("3"), 3, InternalAggregations.EMPTY, false) + ), + false, + Map.of() + ); + InternalTimeSeries third = new InternalTimeSeries( + "ts", + List.of( + new InternalBucket(new BytesRef("1"), 2, InternalAggregations.EMPTY, false), + new InternalBucket(new BytesRef("3"), 4, InternalAggregations.EMPTY, false), + new InternalBucket(new BytesRef("9"), 4, InternalAggregations.EMPTY, false) + ), + false, + Map.of() + ); + AggregationReduceContext context = new AggregationReduceContext.ForFinal( + new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()), + mockScriptService(), + () -> false, + new TimeSeriesAggregationBuilder("ts"), + value -> {}, + PipelineAggregator.PipelineTree.EMPTY + ); + + InternalTimeSeries result = (InternalTimeSeries) first.reduce(List.of(first, second, third), context); + assertThat(result.getBuckets().get(0).key.utf8ToString(), equalTo("1")); + assertThat(result.getBuckets().get(0).getDocCount(), equalTo(5L)); + assertThat(result.getBuckets().get(1).key.utf8ToString(), equalTo("10")); + assertThat(result.getBuckets().get(1).getDocCount(), equalTo(6L)); + assertThat(result.getBuckets().get(2).key.utf8ToString(), equalTo("2")); + assertThat(result.getBuckets().get(2).getDocCount(), equalTo(3L)); + assertThat(result.getBuckets().get(3).key.utf8ToString(), equalTo("3")); + assertThat(result.getBuckets().get(3).getDocCount(), equalTo(7L)); + assertThat(result.getBuckets().get(4).key.utf8ToString(), equalTo("9")); + assertThat(result.getBuckets().get(4).getDocCount(), equalTo(9L)); + } + + @Override + protected InternalTimeSeries mutateInstance(InternalTimeSeries instance) { + return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929 + } +} diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index cf26c21f261c2..f9cab9ff49cbf 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -19,7 +19,12 @@ import org.apache.lucene.search.Query; import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.aggregations.bucket.AggregationTestCase; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; @@ -33,6 +38,7 @@ import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram; import org.elasticsearch.search.aggregations.metrics.Sum; import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.test.index.IndexVersionUtils; import java.io.IOException; import java.util.ArrayList; @@ -187,9 +193,9 @@ public void testAggregationSize() throws IOException { verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=aaa, dim2=xxx}").docCount, equalTo(2L))); verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=aaa, dim2=yyy}").docCount, equalTo(2L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=bbb, dim2=zzz}").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=bbb, dim2=zzz}").docCount, equalTo(4L))); - for (int i = 1; i < 3; i++) { + for (int i = 1; i <= 3; i++) { int size = i; Consumer limitedVerifier = ts -> { assertThat(ts.getBuckets(), hasSize(size)); diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java new file mode 100644 index 0000000000000..7b7c3ee4467a5 --- /dev/null +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java @@ -0,0 +1,320 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.aggregations.bucket.timeseries; + +import org.apache.lucene.document.DoubleDocValuesField; +import org.apache.lucene.document.FloatDocValuesField; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.aggregations.bucket.AggregationTestCase; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper.TimeSeriesIdBuilder; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.test.index.IndexVersionUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + +public class TimeSeriesAggregatorTsidHashingTests extends AggregationTestCase { + + @Override + protected List getSupportedValuesSourceTypes() { + return List.of(); + } + + /** + * @return {@link IndexSettings} including a specific {@link IndexVersion}. Versions up to 8.11.0 do not use tsid hashing and + * {@link InternalTimeSeries#getBucketByKey(String)} works passing dimension name/value pairs. After introducing tsis hashing + * that is not possible anymore and we need to get buckets by the tsid value instead. + */ + @Override + protected IndexSettings createIndexSettings() { + final IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.V_8_10_0, // TODO: change to 8.11.0 after release + IndexVersion.current() + ); + return new IndexSettings( + IndexMetadata.builder("_index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, indexVersion)) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(System.currentTimeMillis()) + .build(), + Settings.EMPTY + ); + } + + public void testStandAloneTimeSeriesWithSum() throws IOException { + TimeSeriesAggregationBuilder aggregationBuilder = new TimeSeriesAggregationBuilder("ts").subAggregation(sum("sum").field("val1")); + long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2021-01-01T00:00:00Z"); + timeSeriesTestCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { + writeTS(iw, startTime + 1, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] { "val1", 1 }); + writeTS(iw, startTime + 2, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] { "val1", 2 }); + writeTS(iw, startTime + 3, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 3 }); + writeTS(iw, startTime + 4, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 4 }); + writeTS(iw, startTime + 5, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] { "val1", 5 }); + writeTS(iw, startTime + 6, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] { "val1", 6 }); + writeTS(iw, startTime + 7, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 7 }); + writeTS(iw, startTime + 8, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 8 }); + }, ts -> { + assertThat(ts.getBuckets(), hasSize(3)); + + assertThat( + ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg}").docCount, + equalTo(2L) + ); + assertThat( + ((Sum) ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg}") + .getAggregations() + .get("sum")).value(), + equalTo(6.0) + ); + assertThat( + ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4}").docCount, + equalTo(2L) + ); + assertThat( + ((Sum) ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4}") + .getAggregations() + .get("sum")).value(), + equalTo(8.0) + ); + assertThat( + ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o}").docCount, + equalTo(4L) + ); + assertThat( + ((Sum) ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o}") + .getAggregations() + .get("sum")).value(), + equalTo(22.0) + ); + + }, + new KeywordFieldMapper.KeywordFieldType("dim1"), + new KeywordFieldMapper.KeywordFieldType("dim2"), + new NumberFieldMapper.NumberFieldType("val1", NumberFieldMapper.NumberType.INTEGER) + ); + } + + public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimensions, Object[] metrics) throws IOException { + final List fields = new ArrayList<>(); + fields.add(new SortedNumericDocValuesField(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp)); + fields.add(new LongPoint(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp)); + final TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(null); + for (int i = 0; i < dimensions.length; i += 2) { + if (dimensions[i + 1] instanceof Number n) { + builder.addLongDimension(dimensions[i].toString(), n.longValue()); + } else { + builder.addKeywordDimension(dimensions[i].toString(), dimensions[i + 1].toString()); + } + } + for (int i = 0; i < metrics.length; i += 2) { + if (metrics[i + 1] instanceof Integer || metrics[i + 1] instanceof Long) { + fields.add(new NumericDocValuesField(metrics[i].toString(), ((Number) metrics[i + 1]).longValue())); + } else if (metrics[i + 1] instanceof Float) { + fields.add(new FloatDocValuesField(metrics[i].toString(), (float) metrics[i + 1])); + } else if (metrics[i + 1] instanceof Double) { + fields.add(new DoubleDocValuesField(metrics[i].toString(), (double) metrics[i + 1])); + } + } + fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.withHash().toBytesRef())); + // TODO: Handle metrics + iw.addDocument(fields); + } + + public void testWithDateHistogramExecutedAsFilterByFilterWithTimeSeriesIndexSearcher() throws IOException { + DateHistogramAggregationBuilder aggregationBuilder = new DateHistogramAggregationBuilder("by_timestamp").field("@timestamp") + .fixedInterval(DateHistogramInterval.HOUR) + .subAggregation(new TimeSeriesAggregationBuilder("ts").subAggregation(sum("sum").field("val1"))); + + // Before this threw a CollectionTerminatedException because FilterByFilterAggregation#getLeafCollector() always returns a + // LeafBucketCollector.NO_OP_COLLECTOR instance. And TimeSeriesIndexSearcher can't deal with this when initializing the + // leaf walkers. + testCase(iw -> { + long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); + for (int i = 1; i <= 5000; i++) { + writeTS(iw, startTime++, new Object[] { "dim1", "aaa" }, new Object[] { "val1", 1 }); + } + }, internalAggregation -> { + InternalDateHistogram dateHistogram = (InternalDateHistogram) internalAggregation; + assertThat(dateHistogram.getBuckets(), hasSize(1)); + InternalTimeSeries timeSeries = dateHistogram.getBuckets().get(0).getAggregations().get("ts"); + assertThat(timeSeries.getBuckets(), hasSize(1)); + Sum sum = timeSeries.getBuckets().get(0).getAggregations().get("sum"); + assertThat(sum.value(), equalTo(5000.0)); + }, + new AggTestConfig( + aggregationBuilder, + TimeSeriesIdFieldMapper.FIELD_TYPE, + new DateFieldMapper.DateFieldType("@timestamp"), + new KeywordFieldMapper.KeywordFieldType("dim1"), + new NumberFieldMapper.NumberFieldType("val1", NumberFieldMapper.NumberType.INTEGER) + ).withQuery(new MatchAllDocsQuery()) + ); + } + + public void testMultiBucketAggregationAsSubAggregation() throws IOException { + long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); + CheckedConsumer buildIndex = iw -> { + writeTS(iw, startTime + 1, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] {}); + writeTS(iw, startTime + 2, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] {}); + writeTS(iw, startTime + 3, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] {}); + writeTS(iw, startTime + 4, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] {}); + writeTS(iw, startTime + 5, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] {}); + writeTS(iw, startTime + 6, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] {}); + writeTS(iw, startTime + 7, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] {}); + writeTS(iw, startTime + 8, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] {}); + }; + Consumer verifier = ts -> { + assertThat(ts.getBuckets(), hasSize(3)); + + assertThat( + ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg}").docCount, + equalTo(2L) + ); + InternalDateHistogram byTimeStampBucket = ts.getBucketByKey( + "{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg}" + ).getAggregations().get("by_timestamp"); + assertThat( + byTimeStampBucket.getBuckets(), + contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) + ); + assertThat( + ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4}").docCount, + equalTo(2L) + ); + byTimeStampBucket = ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4}") + .getAggregations() + .get("by_timestamp"); + assertThat( + byTimeStampBucket.getBuckets(), + contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) + ); + assertThat( + ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o}").docCount, + equalTo(4L) + ); + byTimeStampBucket = ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o}") + .getAggregations() + .get("by_timestamp"); + assertThat( + byTimeStampBucket.getBuckets(), + contains(new InternalDateHistogram.Bucket(startTime, 4, false, null, InternalAggregations.EMPTY)) + ); + }; + + DateHistogramAggregationBuilder dateBuilder = new DateHistogramAggregationBuilder("by_timestamp"); + dateBuilder.field("@timestamp"); + dateBuilder.fixedInterval(DateHistogramInterval.seconds(1)); + TimeSeriesAggregationBuilder tsBuilder = new TimeSeriesAggregationBuilder("by_tsid"); + tsBuilder.subAggregation(dateBuilder); + timeSeriesTestCase(tsBuilder, new MatchAllDocsQuery(), buildIndex, verifier); + } + + public void testAggregationSize() throws IOException { + CheckedConsumer buildIndex = multiTsWriter(); + + List> verifiers = new ArrayList>(); + + verifiers.add( + ts -> assertThat( + ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o}").docCount, + equalTo(4L) + ) + ); + verifiers.add( + ts -> assertThat( + ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg}").docCount, + equalTo(2L) + ) + ); + verifiers.add( + ts -> assertThat( + ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4}").docCount, + equalTo(2L) + ) + ); + + for (int i = 1; i <= 3; i++) { + int size = i; + Consumer limitedVerifier = ts -> { + assertThat(ts.getBuckets(), hasSize(size)); + + for (int j = 0; j < size; j++) { + verifiers.get(j).accept(ts); + } + }; + + TimeSeriesAggregationBuilder limitedTsBuilder = new TimeSeriesAggregationBuilder("by_tsid"); + limitedTsBuilder.setSize(i); + timeSeriesTestCase(limitedTsBuilder, new MatchAllDocsQuery(), buildIndex, limitedVerifier); + } + } + + private CheckedConsumer multiTsWriter() { + long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); + return iw -> { + writeTS(iw, startTime + 1, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] { "val1", 1 }); + writeTS(iw, startTime + 2, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] { "val1", 2 }); + writeTS(iw, startTime + 3, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 3 }); + writeTS(iw, startTime + 4, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 4 }); + writeTS(iw, startTime + 5, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] { "val1", 5 }); + writeTS(iw, startTime + 6, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] { "val1", 6 }); + writeTS(iw, startTime + 7, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 7 }); + writeTS(iw, startTime + 8, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 8 }); + }; + } + + private void timeSeriesTestCase( + TimeSeriesAggregationBuilder builder, + Query query, + CheckedConsumer buildIndex, + Consumer verify, + MappedFieldType... fieldTypes + ) throws IOException { + MappedFieldType[] newFieldTypes = new MappedFieldType[fieldTypes.length + 2]; + newFieldTypes[0] = TimeSeriesIdFieldMapper.FIELD_TYPE; + newFieldTypes[1] = new DateFieldMapper.DateFieldType("@timestamp"); + System.arraycopy(fieldTypes, 0, newFieldTypes, 2, fieldTypes.length); + + testCase(buildIndex, verify, new AggTestConfig(builder, newFieldTypes).withQuery(query)); + } + +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 05b254cb20008..b7ee67b907496 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -162,12 +162,11 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { /** * Decode the {@code _tsid} into a human readable map. */ - public static Map decodeTsid(StreamInput in) { + public static Object decodeTsid(StreamInput in) { try { int sizeOrTsidHashSentinel = in.readVInt(); if (sizeOrTsidHashSentinel == TSID_HASH_SENTINEL) { - final BytesRef bytesRef = in.readBytesRef(); - return Collections.singletonMap("_tsid", Base64.getUrlEncoder().withoutPadding().encodeToString(bytesRef.bytes)); + return Base64.getUrlEncoder().withoutPadding().encodeToString(in.readBytesRef().bytes); } Map result = new LinkedHashMap<>(sizeOrTsidHashSentinel); @@ -195,9 +194,11 @@ public static Map decodeTsid(StreamInput in) { public static class TimeSeriesIdBuilder implements DocumentFields { + private static final int SEED = 0; + public static final int MAX_DIMENSIONS = 512; - private record DimensionDataHolder(BytesRef name, BytesRef value) {} + private record DimensionDataHolder(BytesRef name, BytesReference value) {} private final Murmur3Hasher tsidHasher = new Murmur3Hasher(0); @@ -227,7 +228,7 @@ public BytesReference withoutHash() throws IOException { out.writeVInt(dimensions.size()); for (DimensionDataHolder entry : dimensions) { out.writeBytesRef(entry.name); - out.writeBytesRef(entry.value); + entry.value.writeTo(out); } return out.bytes(); } @@ -268,8 +269,14 @@ public BytesReference withHash() throws IOException { int tsidHashStartIndex = tsidHashIndex; for (final DimensionDataHolder dimension : dimensions) { if ((tsidHashIndex - tsidHashStartIndex) >= 4 * numberOfDimensions) break; + final BytesRef dimensionValueBytesRef = dimension.value.toBytesRef(); ByteUtils.writeIntLE( - StringHelper.murmurhash3_x86_32(dimension.value.bytes, dimension.value.offset, dimension.value.length, 0), + StringHelper.murmurhash3_x86_32( + dimensionValueBytesRef.bytes, + dimensionValueBytesRef.offset, + dimensionValueBytesRef.length, + SEED + ), tsidHash, tsidHashIndex ); @@ -279,7 +286,7 @@ public BytesReference withHash() throws IOException { // NOTE: hash all dimension field allValues tsidHasher.reset(); for (final DimensionDataHolder dimension : dimensions) { - tsidHasher.update(dimension.value.bytes); + tsidHasher.update(dimension.value.toBytesRef().bytes); } tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); @@ -358,7 +365,7 @@ public void addMetric(String fieldName) { } private void add(String fieldName, BytesReference encoded) throws IOException { - final DimensionDataHolder dimension = new DimensionDataHolder(new BytesRef(fieldName), encoded.toBytesRef()); + final DimensionDataHolder dimension = new DimensionDataHolder(new BytesRef(fieldName), encoded); if (dimensions.contains(dimension)) { throw new IllegalArgumentException("Dimension field [" + fieldName + "] cannot be a multi-valued field."); } @@ -366,7 +373,7 @@ private void add(String fieldName, BytesReference encoded) throws IOException { } } - public static Map decodeTsid(BytesRef bytesRef) { + public static Object decodeTsid(BytesRef bytesRef) { try (StreamInput input = new BytesArray(bytesRef).streamInput()) { return decodeTsid(input); } catch (IOException ex) { diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 08625a08723cf..6e17966a1c4da 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -724,7 +724,7 @@ public BytesRef parseBytesRef(Object value) { } try { - return builder.withoutHash().toBytesRef(); + return builder.withHash().toBytesRef(); } catch (IOException e) { throw new IllegalArgumentException(e); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java new file mode 100644 index 0000000000000..2070bc8d4dcf9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java @@ -0,0 +1,275 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.routing.IndexRouting; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; + +public class IdLoaderTsidHashingTests extends ESTestCase { + + public void testSynthesizeIdSimple() throws Exception { + var routingPaths = List.of("dim1"); + var routing = createRouting(routingPaths); + var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); + + long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); + List docs = List.of( + new Doc(startTime, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), + new Doc(startTime + 1, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))), + new Doc(startTime + 2, List.of(new Dimension("dim1", "bbb"), new Dimension("dim2", "xxx"))) + ); + CheckedConsumer verify = indexReader -> { + assertThat(indexReader.leaves(), hasSize(1)); + LeafReader leafReader = indexReader.leaves().get(0).reader(); + assertThat(leafReader.numDocs(), equalTo(3)); + var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 1, 2 }); + assertThat(leaf.getId(0), equalTo(expectedId(routing, docs.get(2)))); + assertThat(leaf.getId(1), equalTo(expectedId(routing, docs.get(0)))); + assertThat(leaf.getId(2), equalTo(expectedId(routing, docs.get(1)))); + }; + prepareIndexReader(indexAndForceMerge(routing, docs), verify, false); + } + + public void testSynthesizeIdMultipleSegments() throws Exception { + var routingPaths = List.of("dim1"); + var routing = createRouting(routingPaths); + var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); + + long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); + List docs1 = List.of( + new Doc(startTime, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), + new Doc(startTime - 1, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), + new Doc(startTime - 2, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), + new Doc(startTime - 3, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))) + ); + List docs2 = List.of( + new Doc(startTime, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))), + new Doc(startTime - 1, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))), + new Doc(startTime - 2, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))), + new Doc(startTime - 3, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))) + ); + List docs3 = List.of( + new Doc(startTime, List.of(new Dimension("dim1", "bbb"), new Dimension("dim2", "yyy"))), + new Doc(startTime - 1, List.of(new Dimension("dim1", "bbb"), new Dimension("dim2", "yyy"))), + new Doc(startTime - 2, List.of(new Dimension("dim1", "bbb"), new Dimension("dim2", "yyy"))), + new Doc(startTime - 3, List.of(new Dimension("dim1", "bbb"), new Dimension("dim2", "yyy"))) + ); + CheckedConsumer buildIndex = writer -> { + for (Doc doc : docs1) { + indexDoc(routing, writer, doc); + } + writer.flush(); + for (Doc doc : docs2) { + indexDoc(routing, writer, doc); + } + writer.flush(); + for (Doc doc : docs3) { + indexDoc(routing, writer, doc); + } + writer.flush(); + }; + CheckedConsumer verify = indexReader -> { + assertThat(indexReader.leaves(), hasSize(3)); + { + LeafReader leafReader = indexReader.leaves().get(0).reader(); + assertThat(leafReader.numDocs(), equalTo(docs1.size())); + var leaf = idLoader.leaf(null, leafReader, IntStream.range(0, docs1.size()).toArray()); + for (int i = 0; i < docs1.size(); i++) { + assertThat(leaf.getId(i), equalTo(expectedId(routing, docs1.get(i)))); + } + } + { + LeafReader leafReader = indexReader.leaves().get(1).reader(); + assertThat(leafReader.numDocs(), equalTo(docs2.size())); + var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 3 }); + assertThat(leaf.getId(0), equalTo(expectedId(routing, docs2.get(0)))); + assertThat(leaf.getId(3), equalTo(expectedId(routing, docs2.get(3)))); + } + { + LeafReader leafReader = indexReader.leaves().get(2).reader(); + assertThat(leafReader.numDocs(), equalTo(docs3.size())); + var leaf = idLoader.leaf(null, leafReader, new int[] { 1, 2 }); + assertThat(leaf.getId(1), equalTo(expectedId(routing, docs3.get(1)))); + assertThat(leaf.getId(2), equalTo(expectedId(routing, docs3.get(2)))); + } + { + LeafReader leafReader = indexReader.leaves().get(2).reader(); + assertThat(leafReader.numDocs(), equalTo(docs3.size())); + var leaf = idLoader.leaf(null, leafReader, new int[] { 3 }); + expectThrows(IllegalArgumentException.class, () -> leaf.getId(0)); + } + }; + prepareIndexReader(buildIndex, verify, true); + } + + public void testSynthesizeIdRandom() throws Exception { + var routingPaths = List.of("dim1"); + var routing = createRouting(routingPaths); + var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); + + long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); + Set expectedIDs = new HashSet<>(); + List randomDocs = new ArrayList<>(); + int numberOfTimeSeries = randomIntBetween(8, 64); + for (int i = 0; i < numberOfTimeSeries; i++) { + int numberOfDimensions = randomIntBetween(1, 6); + List dimensions = new ArrayList<>(numberOfDimensions); + for (int j = 1; j <= numberOfDimensions; j++) { + String fieldName = "dim" + j; + Object value; + if (j == 5) { + value = randomLongBetween(1, 20); + } else { + value = randomAlphaOfLength(4); + } + dimensions.add(new Dimension(fieldName, value)); + } + int numberOfSamples = randomIntBetween(1, 16); + for (int j = 0; j < numberOfSamples; j++) { + Doc doc = new Doc(startTime++, dimensions); + randomDocs.add(doc); + expectedIDs.add(expectedId(routing, doc)); + } + } + CheckedConsumer verify = indexReader -> { + assertThat(indexReader.leaves(), hasSize(1)); + LeafReader leafReader = indexReader.leaves().get(0).reader(); + assertThat(leafReader.numDocs(), equalTo(randomDocs.size())); + var leaf = idLoader.leaf(null, leafReader, IntStream.range(0, randomDocs.size()).toArray()); + for (int i = 0; i < randomDocs.size(); i++) { + String actualId = leaf.getId(i); + assertTrue("docId=" + i + " id=" + actualId, expectedIDs.remove(actualId)); + } + }; + prepareIndexReader(indexAndForceMerge(routing, randomDocs), verify, false); + assertThat(expectedIDs, empty()); + } + + private static CheckedConsumer indexAndForceMerge(IndexRouting.ExtractFromSource routing, List docs) { + return writer -> { + for (Doc doc : docs) { + indexDoc(routing, writer, doc); + } + writer.forceMerge(1); + }; + } + + private void prepareIndexReader( + CheckedConsumer buildIndex, + CheckedConsumer verify, + boolean noMergePolicy + ) throws IOException { + try (Directory directory = newDirectory()) { + IndexWriterConfig config = LuceneTestCase.newIndexWriterConfig(random(), new MockAnalyzer(random())); + if (noMergePolicy) { + config.setMergePolicy(NoMergePolicy.INSTANCE); + config.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); + } + Sort sort = new Sort( + new SortField(TimeSeriesIdFieldMapper.NAME, SortField.Type.STRING, false), + new SortedNumericSortField(DataStreamTimestampFieldMapper.DEFAULT_PATH, SortField.Type.LONG, true) + ); + config.setIndexSort(sort); + IndexWriter indexWriter = new IndexWriter(directory, config); + buildIndex.accept(indexWriter); + indexWriter.close(); + + try (DirectoryReader indexReader = DirectoryReader.open(directory);) { + verify.accept(indexReader); + } + } + } + + private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter iw, Doc doc) throws IOException { + final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routing.builder()); + + final List fields = new ArrayList<>(); + fields.add(new SortedNumericDocValuesField(DataStreamTimestampFieldMapper.DEFAULT_PATH, doc.timestamp)); + fields.add(new LongPoint(DataStreamTimestampFieldMapper.DEFAULT_PATH, doc.timestamp)); + for (Dimension dimension : doc.dimensions) { + if (dimension.value instanceof Number n) { + builder.addLongDimension(dimension.field, n.longValue()); + fields.add(new SortedNumericDocValuesField(dimension.field, ((Number) dimension.value).longValue())); + } else { + builder.addKeywordDimension(dimension.field, dimension.value.toString()); + fields.add(new SortedSetDocValuesField(dimension.field, new BytesRef(dimension.value.toString()))); + } + } + BytesRef tsid = builder.withHash().toBytesRef(); + fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, tsid)); + iw.addDocument(fields); + } + + private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc) throws IOException { + var routingBuilder = routing.builder(); + var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routingBuilder); + for (Dimension dimension : doc.dimensions) { + if (dimension.value instanceof Number n) { + timeSeriesIdBuilder.addLongDimension(dimension.field, n.longValue()); + } else { + timeSeriesIdBuilder.addKeywordDimension(dimension.field, dimension.value.toString()); + } + } + return TsidExtractingIdFieldMapper.createId( + false, + routingBuilder, + timeSeriesIdBuilder.withHash().toBytesRef(), + doc.timestamp, + new byte[16] + ); + } + + private static IndexRouting.ExtractFromSource createRouting(List routingPaths) { + var settings = indexSettings(IndexVersion.current(), 2, 1).put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2000-01-01T00:00:00.000Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2001-01-01T00:00:00.000Z") + .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPaths) + .build(); + var indexMetadata = IndexMetadata.builder("index").settings(settings).build(); + return (IndexRouting.ExtractFromSource) IndexRouting.fromIndexMetadata(indexMetadata); + } + + record Doc(long timestamp, List dimensions) {} + + record Dimension(String field, Object value) {} + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 483b60754cef2..c4227823d2391 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -15,9 +15,13 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.List; import java.util.Map; import static org.elasticsearch.test.MapMatcher.assertMap; @@ -45,6 +49,15 @@ protected void registerParameters(ParameterChecker checker) throws IOException { // There aren't any parameters } + @Override + protected IndexVersion getVersion() { + return IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.V_8_8_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_ID_HASHING) + ); + } + private DocumentMapper createDocumentMapper(String routingPath, XContentBuilder mappings) throws IOException { return createMapperService( getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) @@ -70,6 +83,7 @@ private static BytesRef parseAndGetTsid(DocumentMapper docMapper, CheckedConsume return parseDocument(docMapper, f).rootDoc().getBinaryValue(TimeSeriesIdFieldMapper.NAME); } + @SuppressWarnings("unchecked") public void testEnabledInTimeSeriesMode() throws Exception { DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -77,9 +91,15 @@ public void testEnabledInTimeSeriesMode() throws Exception { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100).field("c", 500)); + assertThat( + doc.rootDoc().getBinaryValue("_tsid"), + equalTo(new BytesRef("\u0002\u0001as\u0005value\u0001bl\u0000\u0000\u0000\u0000\u0000\u0000\u0000d")) + ); + assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); + assertThat(doc.rootDoc().getField("b").numericValue(), equalTo(100L)); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "30CO74tuRyatuMXEvNJvbgAAAAAAAAAAAAAAAAAAAADuDjjxPwT2Ol0UAzaqxX0Rpx75AkZHRdY") + (Map) TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + matchesMap().entry("a", "value").entry("b", 100L) ); } @@ -107,6 +127,7 @@ public void testIncludeInDocumentNotAllowed() throws Exception { /** * Test with non-randomized string for sanity checking. */ + @SuppressWarnings("unchecked") public void testStrings() throws IOException { DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -125,11 +146,12 @@ public void testStrings() throws IOException { b -> b.field("a", "foo").field("b", "bar").field("c", "baz").startObject("o").field("e", "bort").endObject() ); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "S3oqtElkKwgW-mWA1nr9gAAAAAAAAAAAAAAAAAAAAACv9houtA-WtALkiErAWD5o8PqlkO2CPtw") + (Map) TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + matchesMap().entry("a", "foo").entry("o.e", "bort") ); } + @SuppressWarnings("unchecked") public void testUnicodeKeys() throws IOException { String fire = new String(new int[] { 0x1F525 }, 0, 1); String coffee = "\u2615"; @@ -139,12 +161,15 @@ public void testUnicodeKeys() throws IOException { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field(fire, "hot").field(coffee, "good")); - Map tsid = TimeSeriesIdFieldMapper.decodeTsid( + Map tsid = (Map) TimeSeriesIdFieldMapper.decodeTsid( new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes) ); - assertMap(tsid, matchesMap().entry("_tsid", "A-nLhW-M69syDAiaaHLtCgAAAAAAAAAAAAAAAAAAAACejAXjDKlQ6oAVV6S1YhlDnYAgGZ4kqeg")); + assertMap(tsid, matchesMap().entry(coffee, "good").entry(fire, "hot")); + // Also make sure the keys are in order + assertThat(List.copyOf(tsid.keySet()), equalTo(List.of(coffee, fire))); } + @SuppressWarnings("unchecked") public void testKeywordTooLong() throws IOException { DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -152,11 +177,12 @@ public void testKeywordTooLong() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "FlVJQe85E3Mv4Ekz_gxSawAAAAAAAAAAAAAAAAAAAAC9ZDUoCO2eynkD-lZbEGc5huKIYw") + (Map) TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + matchesMap().entry("a", "more_than_1024_bytes".repeat(52)) ); } + @SuppressWarnings("unchecked") public void testKeywordTooLongUtf8() throws IOException { DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -165,8 +191,8 @@ public void testKeywordTooLongUtf8() throws IOException { String theWordLong = "長い"; ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "FlVJQe85E3Mv4Ekz_gxSawAAAAAAAAAAAAAAAAAAAABM43f6n6D4bFDIodLIvFw5j_1Vew") + (Map) TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + matchesMap().entry("a", theWordLong.repeat(200)) ); } @@ -184,6 +210,7 @@ public void testKeywordNull() throws IOException { /** * Test with non-randomized longs for sanity checking. */ + @SuppressWarnings("unchecked") public void testLong() throws IOException { DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -206,8 +233,8 @@ public void testLong() throws IOException { b.startObject("o").field("e", 1234).endObject(); }); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJy91wa2A3yf66JQYgw9_3GPhrunFA") + (Map) TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", 1234L) ); } @@ -238,6 +265,7 @@ public void testLongNull() throws IOException { /** * Test with non-randomized integers for sanity checking. */ + @SuppressWarnings("unchecked") public void testInteger() throws IOException { DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -260,8 +288,8 @@ public void testInteger() throws IOException { b.startObject("o").field("e", Integer.MIN_VALUE).endObject(); }); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJy9XyEVAdPjbPnyFfNkppmgRrzARX") + (Map) TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Integer.MIN_VALUE) ); } @@ -296,6 +324,7 @@ public void testIntegerOutOfRange() throws IOException { /** * Test with non-randomized shorts for sanity checking. */ + @SuppressWarnings("unchecked") public void testShort() throws IOException { DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -318,8 +347,8 @@ public void testShort() throws IOException { b.startObject("o").field("e", Short.MIN_VALUE).endObject(); }); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJyyYAWY_JC1ss-P30niOYXA2x9kLq") + (Map) TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Short.MIN_VALUE) ); } @@ -354,6 +383,7 @@ public void testShortOutOfRange() throws IOException { /** * Test with non-randomized shorts for sanity checking. */ + @SuppressWarnings("unchecked") public void testByte() throws IOException { DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -376,8 +406,8 @@ public void testByte() throws IOException { b.startObject("o").field("e", (int) Byte.MIN_VALUE).endObject(); }); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("_tsid", "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJyygrqocRYcbJqQJZlIyTzrY9S9Nt") + (Map) TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Byte.MIN_VALUE) ); } @@ -412,6 +442,7 @@ public void testByteOutOfRange() throws IOException { /** * Test with non-randomized ips for sanity checking. */ + @SuppressWarnings("unchecked") public void testIp() throws IOException { DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -434,8 +465,8 @@ public void testIp() throws IOException { b.startObject("o").field("e", "255.255.255.1").endObject(); }); assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("_tsid", "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAAALrOlWSrpJy7PzTuwgaqmbsw6Oz00Ir_8Cw9Xm") + (Map) TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + matchesMap().entry("kw", "kw").entry("a", "192.168.0.1").entry("o.e", "255.255.255.1") ); } @@ -454,6 +485,7 @@ public void testIpInvalidString() throws IOException { /** * Tests when the total of the tsid is more than 32k. */ + @SuppressWarnings("unchecked") public void testVeryLarge() throws IOException { DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); @@ -469,13 +501,17 @@ public void testVeryLarge() throws IOException { b.field("d" + i, large); } }); - assertMap( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry( - "_tsid", - "QZ1K9tk9UIcpA826eU6H-QAAAAAAAAAAAAAAAAAAAACv9houWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk8VBv2rLPN9GYxF8e8uXojI" - ) + + Map tsidAsMap = (Map) TimeSeriesIdFieldMapper.decodeTsid( + new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes) ); + for (final Map.Entry entry : tsidAsMap.entrySet()) { + if ("b".equals(entry.getKey())) { + assertEquals("foo", entry.getValue()); + } else { + assertEquals(entry.getValue(), large); + } + } } /** @@ -610,7 +646,8 @@ public void testDifferentMetricNamesSameValues() throws IOException { double metricValue = randomDoubleBetween(10, 20, true); ParsedDocument doc1 = parseDocument(docMapper1, d -> d.field("a", "value").field("b", 10).field("m1", metricValue)); ParsedDocument doc2 = parseDocument(docMapper2, d -> d.field("a", "value").field("b", 10).field("m2", metricValue)); - assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + // NOTE: plain tsid (not hashed) does not take metric names/values into account + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java new file mode 100644 index 0000000000000..66af56e67460c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java @@ -0,0 +1,671 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.ByteArrayStreamInput; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.test.index.IndexVersionUtils; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; + +public class TimeSeriesIdFieldMapperTsidHashingTests extends MetadataMapperTestCase { + + @Override + protected String fieldName() { + return TimeSeriesIdFieldMapper.NAME; + } + + @Override + protected boolean isConfigurable() { + return false; + } + + @Override + protected void registerParameters(ParameterChecker checker) throws IOException { + // There aren't any parameters + } + + @Override + protected IndexVersion getVersion() { + return IndexVersionUtils.randomVersionBetween(random(), IndexVersions.TIME_SERIES_ID_HASHING, IndexVersion.current()); + } + + private DocumentMapper createDocumentMapper(String routingPath, XContentBuilder mappings) throws IOException { + return createMapperService( + getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) + .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), 200) // Allow tests that use many dimensions + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") + .build(), + mappings + ).documentMapper(); + } + + private static ParsedDocument parseDocument(DocumentMapper docMapper, CheckedConsumer f) + throws IOException { + // Add the @timestamp field required by DataStreamTimestampFieldMapper for all time series indices + return docMapper.parse(source(null, b -> { + f.accept(b); + b.field("@timestamp", "2021-10-01"); + }, null)); + } + + private static BytesRef parseAndGetTsid(DocumentMapper docMapper, CheckedConsumer f) throws IOException { + return parseDocument(docMapper, f).rootDoc().getBinaryValue(TimeSeriesIdFieldMapper.NAME); + } + + @SuppressWarnings("unchecked") + public void testEnabledInTimeSeriesMode() throws Exception { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "long").field("time_series_dimension", true).endObject(); + })); + + ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100).field("c", 500)); + assertEquals( + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + "30CO74tuRyatuMXEvNJvbgAAAAAAAAAAAAAAAAAAAADuDjjxPwT2Ol0UAzaqxX0Rpx75AkZHRdY" + ); + } + + public void testDisabledInStandardMode() throws Exception { + DocumentMapper docMapper = createMapperService( + getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()).build(), + mapping(b -> {}) + ).documentMapper(); + assertThat(docMapper.metadataMapper(TimeSeriesIdFieldMapper.class), is(nullValue())); + + ParsedDocument doc = docMapper.parse(source("id", b -> b.field("field", "value"), null)); + assertThat(doc.rootDoc().getBinaryValue("_tsid"), is(nullValue())); + assertThat(doc.rootDoc().get("field"), equalTo("value")); + } + + public void testIncludeInDocumentNotAllowed() throws Exception { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("_tsid", "foo"))); + + assertThat(e.getCause().getMessage(), containsString("Field [_tsid] is a metadata field and cannot be added inside a document")); + } + + /** + * Test with non-randomized string for sanity checking. + */ + @SuppressWarnings("unchecked") + public void testStrings() throws IOException { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("o") + .startObject("properties") + .startObject("e") + .field("type", "keyword") + .field("time_series_dimension", true) + .endObject() + .endObject() + .endObject(); + })); + + BytesRef tsid = parseAndGetTsid( + docMapper, + b -> b.field("a", "foo").field("b", "bar").field("c", "baz").startObject("o").field("e", "bort").endObject() + ); + assertEquals( + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + "S3oqtElkKwgW-mWA1nr9gAAAAAAAAAAAAAAAAAAAAACv9houtA-WtALkiErAWD5o8PqlkO2CPtw" + ); + } + + @SuppressWarnings("unchecked") + public void testUnicodeKeys() throws IOException { + String fire = new String(new int[] { 0x1F525 }, 0, 1); + String coffee = "\u2615"; + DocumentMapper docMapper = createDocumentMapper(fire + "," + coffee, mapping(b -> { + b.startObject(fire).field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject(coffee).field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + + ParsedDocument doc = parseDocument(docMapper, b -> b.field(fire, "hot").field(coffee, "good")); + assertEquals( + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + "A-nLhW-M69syDAiaaHLtCgAAAAAAAAAAAAAAAAAAAACejAXjDKlQ6oAVV6S1YhlDnYAgGZ4kqeg" + ); + } + + @SuppressWarnings("unchecked") + public void testKeywordTooLong() throws IOException { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + + ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); + assertEquals( + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + "FlVJQe85E3Mv4Ekz_gxSawAAAAAAAAAAAAAAAAAAAAC9ZDUoCO2eynkD-lZbEGc5huKIYw" + ); + } + + @SuppressWarnings("unchecked") + public void testKeywordTooLongUtf8() throws IOException { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + + String theWordLong = "長い"; + ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); + assertEquals( + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + "FlVJQe85E3Mv4Ekz_gxSawAAAAAAAAAAAAAAAAAAAABM43f6n6D4bFDIodLIvFw5j_1Vew" + ); + } + + public void testKeywordNull() throws IOException { + DocumentMapper docMapper = createDocumentMapper("r", mapping(b -> { + b.startObject("r").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + + BytesRef withNull = parseAndGetTsid(docMapper, b -> b.field("r", "foo").field("a", (String) null)); + BytesRef withoutField = parseAndGetTsid(docMapper, b -> b.field("r", "foo")); + assertThat(withNull, equalTo(withoutField)); + } + + /** + * Test with non-randomized longs for sanity checking. + */ + @SuppressWarnings("unchecked") + public void testLong() throws IOException { + DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { + b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("a").field("type", "long").field("time_series_dimension", true).endObject(); + b.startObject("o") + .startObject("properties") + .startObject("e") + .field("type", "long") + .field("time_series_dimension", true) + .endObject() + .endObject() + .endObject(); + })); + + BytesRef tsid = parseAndGetTsid(docMapper, b -> { + b.field("kw", "kw"); + b.field("a", 1L); + b.field("b", -1); + b.field("c", "baz"); + b.startObject("o").field("e", 1234).endObject(); + }); + assertEquals( + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJy91wa2A3yf66JQYgw9_3GPhrunFA" + ); + } + + public void testLongInvalidString() throws IOException { + DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { + b.startObject("a").field("type", "long").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_a_long"))); + assertThat( + e.getMessage(), + // TODO describe the document instead of "null" + equalTo("[1:6] failed to parse field [a] of type [long] in a time series document. Preview of field's value: 'not_a_long'") + ); + } + + public void testLongNull() throws IOException { + DocumentMapper docMapper = createDocumentMapper("r", mapping(b -> { + b.startObject("r").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("a").field("type", "long").field("time_series_dimension", true).endObject(); + })); + + BytesRef withNull = parseAndGetTsid(docMapper, b -> b.field("r", "foo").field("a", (Long) null)); + BytesRef withoutField = parseAndGetTsid(docMapper, b -> b.field("r", "foo")); + assertThat(withNull, equalTo(withoutField)); + } + + /** + * Test with non-randomized integers for sanity checking. + */ + @SuppressWarnings("unchecked") + public void testInteger() throws IOException { + DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { + b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("a").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("o") + .startObject("properties") + .startObject("e") + .field("type", "integer") + .field("time_series_dimension", true) + .endObject() + .endObject() + .endObject(); + })); + + BytesRef tsid = parseAndGetTsid(docMapper, b -> { + b.field("kw", "kw"); + b.field("a", 1L); + b.field("b", -1); + b.field("c", "baz"); + b.startObject("o").field("e", Integer.MIN_VALUE).endObject(); + }); + assertEquals( + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJy9XyEVAdPjbPnyFfNkppmgRrzARX" + ); + } + + public void testIntegerInvalidString() throws IOException { + DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { + b.startObject("a").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_an_int"))); + assertThat( + e.getMessage(), + equalTo("[1:6] failed to parse field [a] of type [integer] in a time series document. Preview of field's value: 'not_an_int'") + ); + } + + public void testIntegerOutOfRange() throws IOException { + DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { + b.startObject("a").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", Long.MAX_VALUE))); + assertThat( + e.getMessage(), + equalTo( + "[1:6] failed to parse field [a] of type [integer] in a time series document. Preview of field's value: '" + + Long.MAX_VALUE + + "'" + ) + ); + } + + /** + * Test with non-randomized shorts for sanity checking. + */ + @SuppressWarnings("unchecked") + public void testShort() throws IOException { + DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { + b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("a").field("type", "short").field("time_series_dimension", true).endObject(); + b.startObject("o") + .startObject("properties") + .startObject("e") + .field("type", "short") + .field("time_series_dimension", true) + .endObject() + .endObject() + .endObject(); + })); + + BytesRef tsid = parseAndGetTsid(docMapper, b -> { + b.field("kw", "kw"); + b.field("a", 1L); + b.field("b", -1); + b.field("c", "baz"); + b.startObject("o").field("e", Short.MIN_VALUE).endObject(); + }); + assertEquals( + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJyyYAWY_JC1ss-P30niOYXA2x9kLq" + ); + } + + public void testShortInvalidString() throws IOException { + DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { + b.startObject("a").field("type", "short").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_a_short"))); + assertThat( + e.getMessage(), + equalTo("[1:6] failed to parse field [a] of type [short] in a time series document. Preview of field's value: 'not_a_short'") + ); + } + + public void testShortOutOfRange() throws IOException { + DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { + b.startObject("a").field("type", "short").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", Long.MAX_VALUE))); + assertThat( + e.getMessage(), + equalTo( + "[1:6] failed to parse field [a] of type [short] in a time series document. Preview of field's value: '" + + Long.MAX_VALUE + + "'" + ) + ); + } + + /** + * Test with non-randomized shorts for sanity checking. + */ + @SuppressWarnings("unchecked") + public void testByte() throws IOException { + DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { + b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("a").field("type", "byte").field("time_series_dimension", true).endObject(); + b.startObject("o") + .startObject("properties") + .startObject("e") + .field("type", "byte") + .field("time_series_dimension", true) + .endObject() + .endObject() + .endObject(); + })); + + BytesRef tsid = parseAndGetTsid(docMapper, b -> { + b.field("kw", "kw"); + b.field("a", 1L); + b.field("b", -1); + b.field("c", "baz"); + b.startObject("o").field("e", (int) Byte.MIN_VALUE).endObject(); + }); + assertEquals( + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), + "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJyygrqocRYcbJqQJZlIyTzrY9S9Nt" + ); + } + + public void testByteInvalidString() throws IOException { + DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { + b.startObject("a").field("type", "byte").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_a_byte"))); + assertThat( + e.getMessage(), + equalTo("[1:6] failed to parse field [a] of type [byte] in a time series document. Preview of field's value: 'not_a_byte'") + ); + } + + public void testByteOutOfRange() throws IOException { + DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { + b.startObject("a").field("type", "byte").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", Long.MAX_VALUE))); + assertThat( + e.getMessage(), + equalTo( + "[1:6] failed to parse field [a] of type [byte] in a time series document. Preview of field's value: '" + + Long.MAX_VALUE + + "'" + ) + ); + } + + /** + * Test with non-randomized ips for sanity checking. + */ + @SuppressWarnings("unchecked") + public void testIp() throws IOException { + DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { + b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("a").field("type", "ip").field("time_series_dimension", true).endObject(); + b.startObject("o") + .startObject("properties") + .startObject("e") + .field("type", "ip") + .field("time_series_dimension", true) + .endObject() + .endObject() + .endObject(); + })); + + ParsedDocument doc = parseDocument(docMapper, b -> { + b.field("kw", "kw"); + b.field("a", "192.168.0.1"); + b.field("b", -1); + b.field("c", "baz"); + b.startObject("o").field("e", "255.255.255.1").endObject(); + }); + assertEquals( + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAAALrOlWSrpJy7PzTuwgaqmbsw6Oz00Ir_8Cw9Xm" + ); + } + + public void testIpInvalidString() throws IOException { + DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { + b.startObject("a").field("type", "ip").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_an_ip"))); + assertThat( + e.getMessage(), + equalTo("[1:6] failed to parse field [a] of type [ip] in a time series document. Preview of field's value: 'not_an_ip'") + ); + } + + /** + * Tests when the total of the tsid is more than 32k. + */ + @SuppressWarnings("unchecked") + public void testVeryLarge() throws IOException { + DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { + b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); + for (int i = 0; i < 100; i++) { + b.startObject("d" + i).field("type", "keyword").field("time_series_dimension", true).endObject(); + } + })); + + String large = "many words ".repeat(80); + ParsedDocument doc = parseDocument(docMapper, b -> { + b.field("b", "foo"); + for (int i = 0; i < 100; i++) { + b.field("d" + i, large); + } + }); + assertEquals( + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), + "QZ1K9tk9UIcpA826eU6H-QAAAAAAAAAAAAAAAAAAAACv9houWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk8VBv2rLPN9GYxF8e8uXojI" + ); + } + + /** + * Sending the same document twice produces the same value. + */ + public void testSameGenConsistentForSameDoc() throws IOException { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("c").field("type", "long").field("time_series_dimension", true).endObject(); + })); + + String a = randomAlphaOfLength(10); + int b = between(1, 100); + int c = between(0, 2); + CheckedConsumer fields = d -> d.field("a", a).field("b", b).field("c", (long) c); + ParsedDocument doc1 = parseDocument(docMapper, fields); + ParsedDocument doc2 = parseDocument(docMapper, fields); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } + + /** + * Non-dimension fields don't influence the value of _tsid. + */ + public void testExtraFieldsDoNotMatter() throws IOException { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("c").field("type", "long").field("time_series_dimension", true).endObject(); + })); + + String a = randomAlphaOfLength(10); + int b = between(1, 100); + int c = between(0, 2); + ParsedDocument doc1 = parseDocument( + docMapper, + d -> d.field("a", a).field("b", b).field("c", (long) c).field("e", between(10, 100)) + ); + ParsedDocument doc2 = parseDocument( + docMapper, + d -> d.field("a", a).field("b", b).field("c", (long) c).field("e", between(50, 200)) + ); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } + + /** + * The order that the dimensions appear in the document do not influence the value of _tsid. + */ + public void testOrderDoesNotMatter() throws IOException { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("c").field("type", "long").field("time_series_dimension", true).endObject(); + })); + + String a = randomAlphaOfLength(10); + int b = between(1, 100); + int c = between(0, 2); + ParsedDocument doc1 = parseDocument(docMapper, d -> d.field("a", a).field("b", b).field("c", (long) c)); + ParsedDocument doc2 = parseDocument(docMapper, d -> d.field("b", b).field("a", a).field("c", (long) c)); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } + + /** + * Dimensions that appear in the mapping but not in the document don't influence the value of _tsid. + */ + public void testUnusedExtraDimensions() throws IOException { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("c").field("type", "long").field("time_series_dimension", true).endObject(); + })); + + String a = randomAlphaOfLength(10); + int b = between(1, 100); + CheckedConsumer fields = d -> d.field("a", a).field("b", b); + ParsedDocument doc1 = parseDocument(docMapper, fields); + ParsedDocument doc2 = parseDocument(docMapper, fields); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } + + /** + * Different values for dimensions change the result. + */ + public void testDifferentValues() throws IOException { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + })); + + String a = randomAlphaOfLength(10); + ParsedDocument doc1 = parseDocument(docMapper, d -> d.field("a", a).field("b", between(1, 100))); + ParsedDocument doc2 = parseDocument(docMapper, d -> d.field("a", a + 1).field("b", between(200, 300))); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } + + public void testSameMetricNamesDifferentValues() throws IOException { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("m1").field("type", "double").field("time_series_metric", "gauge").endObject(); + b.startObject("m2").field("type", "integer").field("time_series_metric", "counter").endObject(); + })); + + ParsedDocument doc1 = parseDocument( + docMapper, + d -> d.field("a", "value") + .field("b", 10) + .field("m1", randomDoubleBetween(100, 200, true)) + .field("m2", randomIntBetween(100, 200)) + ); + ParsedDocument doc2 = parseDocument( + docMapper, + d -> d.field("a", "value").field("b", 10).field("m1", randomDoubleBetween(10, 20, true)).field("m2", randomIntBetween(10, 20)) + ); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } + + public void testDifferentMetricNamesSameValues() throws IOException { + DocumentMapper docMapper1 = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("m1").field("type", "double").field("time_series_metric", "gauge").endObject(); + })); + + DocumentMapper docMapper2 = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("m2").field("type", "double").field("time_series_metric", "gauge").endObject(); + })); + + double metricValue = randomDoubleBetween(10, 20, true); + ParsedDocument doc1 = parseDocument(docMapper1, d -> d.field("a", "value").field("b", 10).field("m1", metricValue)); + ParsedDocument doc2 = parseDocument(docMapper2, d -> d.field("a", "value").field("b", 10).field("m2", metricValue)); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } + + /** + * Two documents with the same *values* but different dimension keys will generate + * different {@code _tsid}s. + */ + public void testDifferentDimensions() throws IOException { + // First doc mapper has dimension fields a and b + DocumentMapper docMapper1 = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + })); + // Second doc mapper has dimension fields a and c + DocumentMapper docMapper2 = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("c").field("type", "integer").field("time_series_dimension", true).endObject(); + })); + + String a = randomAlphaOfLength(10); + int b = between(1, 100); + int c = between(5, 500); + CheckedConsumer fields = d -> d.field("a", a).field("b", b).field("c", c); + ParsedDocument doc1 = parseDocument(docMapper1, fields); + ParsedDocument doc2 = parseDocument(docMapper2, fields); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } + + /** + * Documents with fewer dimensions have a different value. + */ + public void testFewerDimensions() throws IOException { + DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); + b.startObject("c").field("type", "integer").field("time_series_dimension", true).endObject(); + })); + + String a = randomAlphaOfLength(10); + int b = between(1, 100); + int c = between(5, 500); + ParsedDocument doc1 = parseDocument(docMapper, d -> d.field("a", a).field("b", b)); + ParsedDocument doc2 = parseDocument(docMapper, d -> d.field("a", a).field("b", b).field("c", c)); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + } +} From 1044c8a8451959fc2a28534845314dc2838d0dbe Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 22 Nov 2023 17:43:35 +0100 Subject: [PATCH 027/125] fix: use tsid hashing everywhere, adapt downsampling remove decodeTsid dependency --- .../timeseries/InternalTimeSeriesTests.java | 2 +- .../timeseries/TimeSeriesAggregatorTests.java | 96 ++++++++++++++----- .../rest-api-spec/test/delete/70_tsdb.yml | 8 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 56 ++--------- .../elasticsearch/search/DocValueFormat.java | 20 +++- .../index/mapper/IdLoaderTests.java | 13 +-- .../mapper/TimeSeriesIdFieldMapperTests.java | 70 +++----------- .../search/DocValueFormatTests.java | 8 +- .../rate/TimeSeriesRateAggregatorTests.java | 13 ++- .../downsample/DimensionFieldProducer.java | 87 +++++++++++++++++ .../DimensionFieldValueFetcher.java | 55 +++++++++++ .../downsample/DownsampleShardIndexer.java | 27 ++++-- ...DownsampleShardPersistentTaskExecutor.java | 1 + .../downsample/DownsampleShardTaskParams.java | 23 ++++- .../xpack/downsample/FieldValueFetcher.java | 4 +- .../downsample/TransportDownsampleAction.java | 14 ++- .../TransportDownsampleIndexerAction.java | 1 + .../DownsampleActionSingleNodeTests.java | 64 +++++++++---- .../aggregations/GeoLineAggregatorTests.java | 2 +- 19 files changed, 375 insertions(+), 189 deletions(-) create mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java create mode 100644 x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldValueFetcher.java diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java index f8b1dba1d86b3..1d9a48a1fd923 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java @@ -53,7 +53,7 @@ private List randomBuckets(boolean keyed, InternalAggregations a builder.addKeywordDimension(entry.getKey(), (String) entry.getValue()); } try { - var key = builder.withoutHash().toBytesRef(); + var key = builder.withHash().toBytesRef(); bucketList.add(new InternalBucket(key, docCount, aggregations, keyed)); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index f9cab9ff49cbf..4326806c90e5e 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -19,12 +19,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.aggregations.bucket.AggregationTestCase; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; @@ -38,7 +33,6 @@ import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram; import org.elasticsearch.search.aggregations.metrics.Sum; import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.test.index.IndexVersionUtils; import java.io.IOException; import java.util.ArrayList; @@ -72,12 +66,36 @@ public void testStandAloneTimeSeriesWithSum() throws IOException { }, ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat(ts.getBucketByKey("{dim1=aaa, dim2=xxx}").docCount, equalTo(2L)); - assertThat(((Sum) ts.getBucketByKey("{dim1=aaa, dim2=xxx}").getAggregations().get("sum")).value(), equalTo(6.0)); - assertThat(ts.getBucketByKey("{dim1=aaa, dim2=yyy}").docCount, equalTo(2L)); - assertThat(((Sum) ts.getBucketByKey("{dim1=aaa, dim2=yyy}").getAggregations().get("sum")).value(), equalTo(8.0)); - assertThat(ts.getBucketByKey("{dim1=bbb, dim2=zzz}").docCount, equalTo(4L)); - assertThat(((Sum) ts.getBucketByKey("{dim1=bbb, dim2=zzz}").getAggregations().get("sum")).value(), equalTo(22.0)); + assertThat( + ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, + equalTo(4L) + ); + assertThat( + ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o") + .getAggregations() + .get("sum")).value(), + equalTo(22.0) + ); + assertThat( + ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, + equalTo(2L) + ); + assertThat( + ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg") + .getAggregations() + .get("sum")).value(), + equalTo(6.0) + ); + assertThat( + ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, + equalTo(2L) + ); + assertThat( + ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4") + .getAggregations() + .get("sum")).value(), + equalTo(8.0) + ); }, new KeywordFieldMapper.KeywordFieldType("dim1"), @@ -107,7 +125,7 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens fields.add(new DoubleDocValuesField(metrics[i].toString(), (double) metrics[i + 1])); } } - fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.withoutHash().toBytesRef())); + fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.withHash().toBytesRef())); // TODO: Handle metrics iw.addDocument(fields); } @@ -158,23 +176,38 @@ public void testMultiBucketAggregationAsSubAggregation() throws IOException { Consumer verifier = ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat(ts.getBucketByKey("{dim1=aaa, dim2=xxx}").docCount, equalTo(2L)); - InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("{dim1=aaa, dim2=xxx}").getAggregations().get("by_timestamp"); + assertThat( + ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, + equalTo(4L) + ); + InternalDateHistogram byTimeStampBucket = ts.getBucketByKey( + "C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o" + ).getAggregations().get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), - contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) + contains(new InternalDateHistogram.Bucket(startTime, 4, false, null, InternalAggregations.EMPTY)) ); - assertThat(ts.getBucketByKey("{dim1=aaa, dim2=yyy}").docCount, equalTo(2L)); - byTimeStampBucket = ts.getBucketByKey("{dim1=aaa, dim2=yyy}").getAggregations().get("by_timestamp"); + assertThat( + ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, + equalTo(2L) + ); + byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg") + .getAggregations() + .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); - assertThat(ts.getBucketByKey("{dim1=bbb, dim2=zzz}").docCount, equalTo(4L)); - byTimeStampBucket = ts.getBucketByKey("{dim1=bbb, dim2=zzz}").getAggregations().get("by_timestamp"); + assertThat( + ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, + equalTo(2L) + ); + byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4") + .getAggregations() + .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), - contains(new InternalDateHistogram.Bucket(startTime, 4, false, null, InternalAggregations.EMPTY)) + contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); }; @@ -191,9 +224,24 @@ public void testAggregationSize() throws IOException { List> verifiers = new ArrayList>(); - verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=aaa, dim2=xxx}").docCount, equalTo(2L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=aaa, dim2=yyy}").docCount, equalTo(2L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=bbb, dim2=zzz}").docCount, equalTo(4L))); + verifiers.add( + ts -> assertThat( + ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, + equalTo(4L) + ) + ); + verifiers.add( + ts -> assertThat( + ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, + equalTo(2L) + ) + ); + verifiers.add( + ts -> assertThat( + ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, + equalTo(2L) + ) + ); for (int i = 1; i <= 3; i++) { int size = i; diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml index 130f3690bb298..6f761f2d4a5e1 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml @@ -49,15 +49,15 @@ setup: location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhC8WO3aVdhvtAAABiHD35_g } + - match: { _id: crxuhDzuYSDy965GAAABiHD35_g } - match: { result: created } - match: { _version: 1 } - do: delete: index: weather_sensors - id: crxuhC8WO3aVdhvtAAABiHD35_g - - match: { _id: crxuhC8WO3aVdhvtAAABiHD35_g } + id: crxuhDzuYSDy965GAAABiHD35_g + - match: { _id: crxuhDzuYSDy965GAAABiHD35_g } - match: { result: deleted } - match: { _version: 2 } @@ -74,6 +74,6 @@ setup: location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhC8WO3aVdhvtAAABiHD35_g } + - match: { _id: crxuhDzuYSDy965GAAABiHD35_g } - match: { result: created } - match: { _version: 3 } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index b7ee67b907496..62a5d937ffd05 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -13,7 +13,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; import org.elasticsearch.cluster.routing.IndexRouting; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.hash.Murmur3Hasher; import org.elasticsearch.common.hash.MurmurHash3; @@ -37,13 +36,10 @@ import java.io.IOException; import java.net.InetAddress; -import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.util.Base64; import java.util.Collections; import java.util.Comparator; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -58,9 +54,7 @@ public class TimeSeriesIdFieldMapper extends MetadataFieldMapper { public static final String CONTENT_TYPE = "_tsid"; public static final TimeSeriesIdFieldType FIELD_TYPE = new TimeSeriesIdFieldType(); public static final TimeSeriesIdFieldMapper INSTANCE = new TimeSeriesIdFieldMapper(); - - // NOTE: used by {@link TimeSeriesIdFieldMapper#decodeTsid(StreamInput)} )}. Remove both if using _tsid hashing - public static final int TSID_HASH_SENTINEL = 0xBAADCAFE; + private static final Base64.Encoder BASE64_ENCODER = Base64.getUrlEncoder().withoutPadding(); @Override public FieldMapper.Builder getMergeBuilder() { @@ -164,31 +158,9 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { */ public static Object decodeTsid(StreamInput in) { try { - int sizeOrTsidHashSentinel = in.readVInt(); - if (sizeOrTsidHashSentinel == TSID_HASH_SENTINEL) { - return Base64.getUrlEncoder().withoutPadding().encodeToString(in.readBytesRef().bytes); - } - Map result = new LinkedHashMap<>(sizeOrTsidHashSentinel); - - for (int i = 0; i < sizeOrTsidHashSentinel; i++) { - String name = in.readBytesRef().utf8ToString(); - - int type = in.read(); - switch (type) { - case (byte) 's' -> // parse a string - result.put(name, in.readBytesRef().utf8ToString()); - case (byte) 'l' -> // parse a long - result.put(name, in.readLong()); - case (byte) 'u' -> { // parse an unsigned_long - Object ul = DocValueFormat.UNSIGNED_LONG_SHIFTED.format(in.readLong()); - result.put(name, ul); - } - default -> throw new IllegalArgumentException("Cannot parse [" + name + "]: Unknown type [" + type + "]"); - } - } - return result; - } catch (IOException | IllegalArgumentException e) { - throw new IllegalArgumentException("Error formatting " + NAME + ": " + e.getMessage(), e); + return base64Encode(in.readBytesRef()); + } catch (IOException e) { + throw new IllegalArgumentException("Unable to read tsid"); } } @@ -250,7 +222,7 @@ public BytesReference withHash() throws IOException { // NOTE: hash all dimension field names int numberOfDimensions = Math.min(MAX_DIMENSIONS, dimensions.size()); int tsidHashIndex = 0; - byte[] tsidHash = new byte[16 + 16 + 16 + 4 * numberOfDimensions]; + byte[] tsidHash = new byte[16 + 16 + 4 * numberOfDimensions]; tsidHasher.reset(); for (final DimensionDataHolder dimension : dimensions) { @@ -258,13 +230,6 @@ public BytesReference withHash() throws IOException { } tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); - // NOTE: hash all metric field names - tsidHasher.reset(); - for (final String metric : metrics) { - tsidHasher.update(metric.getBytes(StandardCharsets.UTF_8)); - } - tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); - // NOTE: concatenate all dimension value hashes up to a certain number of dimensions int tsidHashStartIndex = tsidHashIndex; for (final DimensionDataHolder dimension : dimensions) { @@ -292,7 +257,6 @@ public BytesReference withHash() throws IOException { assert tsidHashIndex == tsidHash.length; try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(TSID_HASH_SENTINEL); out.writeBytesRef(new BytesRef(tsidHash, 0, tsidHash.length)); return out.bytes(); } @@ -374,10 +338,10 @@ private void add(String fieldName, BytesReference encoded) throws IOException { } public static Object decodeTsid(BytesRef bytesRef) { - try (StreamInput input = new BytesArray(bytesRef).streamInput()) { - return decodeTsid(input); - } catch (IOException ex) { - throw new IllegalArgumentException("Dimension field cannot be deserialized.", ex); - } + return base64Encode(bytesRef); + } + + private static String base64Encode(BytesRef bytesRef) { + return BASE64_ENCODER.encodeToString(bytesRef.bytes); } } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 6e17966a1c4da..1ff067f24f29b 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -11,7 +11,6 @@ import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.util.BytesRef; import org.elasticsearch.TransportVersions; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -21,7 +20,6 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper.TimeSeriesIdBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; @@ -694,11 +692,25 @@ public String toString() { @Override public Object format(BytesRef value) { - return TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(value).streamInput()); + return Base64.getUrlEncoder().withoutPadding().encodeToString(value.bytes); } @Override public BytesRef parseBytesRef(Object value) { + if (value instanceof BytesRef) { + return (BytesRef) value; + } + return plainTsidParseBytesRef(value); + } + + /** + * After introducing tsid hashing this tsid parsing logic is deprecated. + * Tsid hashing does not allow us to parse the tsid extracting dimension fields key/values pairs. + * @param value The Map encoding tsid dimension fields key/value pairs. + * + * @return + */ + private BytesRef plainTsidParseBytesRef(Object value) { if (value instanceof Map == false) { throw new IllegalArgumentException("Cannot parse tsid object [" + value + "]"); } @@ -724,7 +736,7 @@ public BytesRef parseBytesRef(Object value) { } try { - return builder.withHash().toBytesRef(); + return builder.withoutHash().toBytesRef(); } catch (IOException e) { throw new IllegalArgumentException(e); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index 8f6b5fb452de7..c94e0dd14ace0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -53,8 +53,8 @@ public void testSynthesizeIdSimple() throws Exception { long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs = List.of( - new Doc(startTime, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), - new Doc(startTime + 1, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))), + new Doc(startTime, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))), + new Doc(startTime + 1, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), new Doc(startTime + 2, List.of(new Dimension("dim1", "bbb"), new Dimension("dim2", "xxx"))) ); CheckedConsumer verify = indexReader -> { @@ -62,9 +62,10 @@ public void testSynthesizeIdSimple() throws Exception { LeafReader leafReader = indexReader.leaves().get(0).reader(); assertThat(leafReader.numDocs(), equalTo(3)); var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 1, 2 }); - assertThat(leaf.getId(0), equalTo(expectedId(routing, docs.get(0)))); + // NOTE: time series data is ordered by (tsid, timestamp) + assertThat(leaf.getId(0), equalTo(expectedId(routing, docs.get(2)))); assertThat(leaf.getId(1), equalTo(expectedId(routing, docs.get(1)))); - assertThat(leaf.getId(2), equalTo(expectedId(routing, docs.get(2)))); + assertThat(leaf.getId(2), equalTo(expectedId(routing, docs.get(0)))); }; prepareIndexReader(indexAndForceMerge(routing, docs), verify, false); } @@ -234,7 +235,7 @@ private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter fields.add(new SortedSetDocValuesField(dimension.field, new BytesRef(dimension.value.toString()))); } } - BytesRef tsid = builder.withoutHash().toBytesRef(); + BytesRef tsid = builder.withHash().toBytesRef(); fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, tsid)); iw.addDocument(fields); } @@ -252,7 +253,7 @@ private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc return TsidExtractingIdFieldMapper.createId( false, routingBuilder, - timeSeriesIdBuilder.withoutHash().toBytesRef(), + timeSeriesIdBuilder.withHash().toBytesRef(), doc.timestamp, new byte[16] ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index c4227823d2391..6ec638a6039b1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -21,11 +21,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; -import java.util.List; -import java.util.Map; -import static org.elasticsearch.test.MapMatcher.assertMap; -import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -97,10 +93,7 @@ public void testEnabledInTimeSeriesMode() throws Exception { ); assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); assertThat(doc.rootDoc().getField("b").numericValue(), equalTo(100L)); - assertMap( - (Map) TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("a", "value").entry("b", 100L) - ); + assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWE"); } public void testDisabledInStandardMode() throws Exception { @@ -145,10 +138,7 @@ public void testStrings() throws IOException { docMapper, b -> b.field("a", "foo").field("b", "bar").field("c", "baz").startObject("o").field("e", "bort").endObject() ); - assertMap( - (Map) TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("a", "foo").entry("o.e", "bort") - ); + assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), "AWE"); } @SuppressWarnings("unchecked") @@ -161,12 +151,8 @@ public void testUnicodeKeys() throws IOException { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field(fire, "hot").field(coffee, "good")); - Map tsid = (Map) TimeSeriesIdFieldMapper.decodeTsid( - new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes) - ); - assertMap(tsid, matchesMap().entry(coffee, "good").entry(fire, "hot")); - // Also make sure the keys are in order - assertThat(List.copyOf(tsid.keySet()), equalTo(List.of(coffee, fire))); + Object tsid = TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); + assertEquals(tsid, "A-I"); } @SuppressWarnings("unchecked") @@ -176,10 +162,7 @@ public void testKeywordTooLong() throws IOException { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); - assertMap( - (Map) TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("a", "more_than_1024_bytes".repeat(52)) - ); + assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); } @SuppressWarnings("unchecked") @@ -190,10 +173,7 @@ public void testKeywordTooLongUtf8() throws IOException { String theWordLong = "長い"; ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); - assertMap( - (Map) TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("a", theWordLong.repeat(200)) - ); + assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); } public void testKeywordNull() throws IOException { @@ -232,10 +212,7 @@ public void testLong() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", 1234).endObject(); }); - assertMap( - (Map) TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", 1234L) - ); + assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testLongInvalidString() throws IOException { @@ -287,10 +264,7 @@ public void testInteger() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", Integer.MIN_VALUE).endObject(); }); - assertMap( - (Map) TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Integer.MIN_VALUE) - ); + assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testIntegerInvalidString() throws IOException { @@ -346,10 +320,7 @@ public void testShort() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", Short.MIN_VALUE).endObject(); }); - assertMap( - (Map) TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Short.MIN_VALUE) - ); + assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testShortInvalidString() throws IOException { @@ -405,10 +376,7 @@ public void testByte() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", (int) Byte.MIN_VALUE).endObject(); }); - assertMap( - (Map) TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - matchesMap().entry("kw", "kw").entry("a", 1L).entry("o.e", (long) Byte.MIN_VALUE) - ); + assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testByteInvalidString() throws IOException { @@ -464,10 +432,7 @@ public void testIp() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", "255.255.255.1").endObject(); }); - assertMap( - (Map) TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - matchesMap().entry("kw", "kw").entry("a", "192.168.0.1").entry("o.e", "255.255.255.1") - ); + assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWFz"); } public void testIpInvalidString() throws IOException { @@ -502,16 +467,11 @@ public void testVeryLarge() throws IOException { } }); - Map tsidAsMap = (Map) TimeSeriesIdFieldMapper.decodeTsid( - new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes) + Object tsid = TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); + assertEquals( + tsid, + "AWJzA2ZvbwJkMHPwBm1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd29yZHMgbWFueSB3b3JkcyA" ); - for (final Map.Entry entry : tsidAsMap.entrySet()) { - if ("b".equals(entry.getKey())) { - assertEquals("foo", entry.getValue()); - } else { - assertEquals(entry.getValue(), large); - } - } } /** diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index eac9cee3aee82..c50a42099b037 100644 --- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -377,9 +377,9 @@ public void testParseTsid() throws IOException { timeSeriesIdBuilder.addKeywordDimension("string", randomAlphaOfLength(10)); timeSeriesIdBuilder.addLongDimension("long", randomLong()); timeSeriesIdBuilder.addUnsignedLongDimension("ulong", randomLong()); - BytesRef tsidBytes = timeSeriesIdBuilder.withoutHash().toBytesRef(); - Object tsidFormat = DocValueFormat.TIME_SERIES_ID.format(tsidBytes); - BytesRef tsidParse = DocValueFormat.TIME_SERIES_ID.parseBytesRef(tsidFormat); - assertEquals(tsidBytes, tsidParse); + BytesRef expected = timeSeriesIdBuilder.withHash().toBytesRef(); + Object tsidFormat = DocValueFormat.TIME_SERIES_ID.format(expected); + BytesRef actual = DocValueFormat.TIME_SERIES_ID.parseBytesRef(tsidFormat); + assertEquals(expected, actual); } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index 46099ebba74eb..333fc888a8f0c 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -82,8 +82,13 @@ public void testNestedWithinDateHistogram() throws IOException { Consumer verifier = r -> { assertThat(r.getBuckets(), hasSize(2)); - assertThat(r.getBucketByKey("{dim=1}"), instanceOf(InternalTimeSeries.InternalBucket.class)); - InternalDateHistogram hb = r.getBucketByKey("{dim=1}").getAggregations().get("date"); + assertThat( + r.getBucketByKey("NFFUy14C9UcX3MnFnsFrpf0AAAAAAAAAAAAAAAAAAAAAiw707uTAKMYIIZrxFHeWv4t2N0w"), + instanceOf(InternalTimeSeries.InternalBucket.class) + ); + InternalDateHistogram hb = r.getBucketByKey("NFFUy14C9UcX3MnFnsFrpf0AAAAAAAAAAAAAAAAAAAAAiw707uTAKMYIIZrxFHeWv4t2N0w") + .getAggregations() + .get("date"); { Rate rate = hb.getBuckets().get(1).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((60 - 37 + 14) / 2000.0 * MILLIS_IN_SECOND, 0.00001)); @@ -92,7 +97,7 @@ public void testNestedWithinDateHistogram() throws IOException { Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((37 - 15) / 1000.0 * MILLIS_IN_SECOND, 0.00001)); } - hb = r.getBucketByKey("{dim=2}").getAggregations().get("date"); + hb = r.getBucketByKey("NFFUy14C9UcX3MnFnsFrpf0AAAAAAAAAAAAAAAAAAAAAoUYmO5acXOT4AOJNerMhm2RoK9I").getAggregations().get("date"); { Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((150 - 74) / 1000.0 * MILLIS_IN_SECOND, 0.00001)); @@ -162,7 +167,7 @@ private List docs(long startTimestamp, String dim, long... values) thr private static BytesReference tsid(String dim) throws IOException { TimeSeriesIdFieldMapper.TimeSeriesIdBuilder idBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); idBuilder.addKeywordDimension("dim", dim); - return idBuilder.withoutHash(); + return idBuilder.withHash(); } private Document doc(long timestamp, BytesReference tsid, long counterValue) { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java new file mode 100644 index 0000000000000..54e3bed8d4b32 --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.elasticsearch.index.fielddata.FormattedDocValues; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public class DimensionFieldProducer extends AbstractDownsampleFieldProducer { + private final Dimension dimension; + + DimensionFieldProducer(final String name, final Dimension dimension) { + super(name); + this.dimension = dimension; + } + + static class Dimension { + private final String name; + private Object value; + private boolean isEmpty; + + Dimension(String name) { + this.name = name; + this.isEmpty = true; + } + + public Object value() { + return value; + } + + public String name() { + return name; + } + + void reset() { + value = null; + isEmpty = true; + } + + void collect(final Object value) { + Objects.requireNonNull(value); + if (isEmpty) { + this.value = value; + this.isEmpty = false; + return; + } + if (value.equals(this.value) == false) { + throw new IllegalArgumentException("Dimension value changed without tsid change [" + value + "] != [" + this.value + "]"); + } + } + } + + @Override + public void reset() { + this.dimension.reset(); + } + + @Override + public boolean isEmpty() { + return this.dimension.isEmpty; + } + + @Override + public void collect(FormattedDocValues docValues, int docId) throws IOException { + if (docValues.advanceExact(docId) == false) { + throw new IllegalArgumentException("Unable to collect dimension [" + this.dimension.name + "]"); + } + int docValueCount = docValues.docValueCount(); + for (int i = 0; i < docValueCount; i++) { + this.dimension.collect(docValues.nextValue()); + } + } + + @Override + public void write(XContentBuilder builder) throws IOException { + if (isEmpty() == false) { + builder.field(this.dimension.name, this.dimension.value()); + } + } +} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldValueFetcher.java new file mode 100644 index 0000000000000..c6ef43cfdacfa --- /dev/null +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldValueFetcher.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.downsample; + +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.SearchExecutionContext; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class DimensionFieldValueFetcher extends FieldValueFetcher { + + private final DimensionFieldProducer dimensionFieldProducer = createFieldProducer(); + + protected DimensionFieldValueFetcher(final MappedFieldType fieldType, final IndexFieldData fieldData) { + super(fieldType.name(), fieldType, fieldData); + } + + private DimensionFieldProducer createFieldProducer() { + final String filedName = fieldType.name(); + return new DimensionFieldProducer(filedName, new DimensionFieldProducer.Dimension(filedName)); + } + + @Override + public AbstractDownsampleFieldProducer fieldProducer() { + return this.dimensionFieldProducer; + } + + /** + * Retrieve field value fetchers for a list of dimensions. + */ + static List create(final SearchExecutionContext context, final String[] dimensions) { + List fetchers = new ArrayList<>(); + for (String dimension : dimensions) { + MappedFieldType fieldType = context.getFieldType(dimension); + assert fieldType != null : "Unknown dimension field type for dimension field: [" + dimension + "]"; + + if (context.fieldExistsInIndex(dimension)) { + final IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); + final String fieldName = context.isMultiField(dimension) + ? fieldType.name().substring(0, fieldType.name().lastIndexOf('.')) + : fieldType.name(); + fetchers.add(new DimensionFieldValueFetcher(fieldType, fieldData)); + } + } + return Collections.unmodifiableList(fetchers); + } +} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java index f74bd299916c1..64da8c8a68080 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java @@ -94,6 +94,7 @@ class DownsampleShardIndexer { private final List fieldValueFetchers; private final DownsampleShardTask task; private final DownsampleShardPersistentTaskState state; + private final String[] dimensions; private volatile boolean abort = false; ByteSizeValue downsampleBulkSize = DOWNSAMPLE_BULK_SIZE; ByteSizeValue downsampleMaxBytesInFlight = DOWNSAMPLE_MAX_BYTES_IN_FLIGHT; @@ -107,6 +108,7 @@ class DownsampleShardIndexer { final DownsampleConfig config, final String[] metrics, final String[] labels, + final String[] dimensions, final DownsampleShardPersistentTaskState state ) { this.task = task; @@ -125,13 +127,15 @@ class DownsampleShardIndexer { null, Collections.emptyMap() ); + this.dimensions = dimensions; this.timestampField = (DateFieldMapper.DateFieldType) searchExecutionContext.getFieldType(config.getTimestampField()); this.timestampFormat = timestampField.docValueFormat(null, null); this.rounding = config.createRounding(); - List fetchers = new ArrayList<>(metrics.length + labels.length); + List fetchers = new ArrayList<>(metrics.length + labels.length + dimensions.length); fetchers.addAll(FieldValueFetcher.create(searchExecutionContext, metrics)); fetchers.addAll(FieldValueFetcher.create(searchExecutionContext, labels)); + fetchers.addAll(DimensionFieldValueFetcher.create(searchExecutionContext, dimensions)); this.fieldValueFetchers = Collections.unmodifiableList(fetchers); toClose = null; } finally { @@ -155,7 +159,7 @@ public DownsampleIndexerAction.ShardDownsampleResponse execute() throws IOExcept BulkProcessor2 bulkProcessor = createBulkProcessor(); try (searcher; bulkProcessor) { final TimeSeriesIndexSearcher timeSeriesSearcher = new TimeSeriesIndexSearcher(searcher, List.of(this::checkCancelled)); - TimeSeriesBucketCollector bucketCollector = new TimeSeriesBucketCollector(bulkProcessor); + TimeSeriesBucketCollector bucketCollector = new TimeSeriesBucketCollector(bulkProcessor, this.dimensions); bucketCollector.preCollection(); timeSeriesSearcher.search(initialStateQuery, bucketCollector); } @@ -332,12 +336,12 @@ private class TimeSeriesBucketCollector extends BucketCollector { long lastTimestamp = Long.MAX_VALUE; long lastHistoTimestamp = Long.MAX_VALUE; - TimeSeriesBucketCollector(BulkProcessor2 bulkProcessor) { + TimeSeriesBucketCollector(BulkProcessor2 bulkProcessor, String[] dimensions) { this.bulkProcessor = bulkProcessor; AbstractDownsampleFieldProducer[] fieldProducers = fieldValueFetchers.stream() .map(FieldValueFetcher::fieldProducer) .toArray(AbstractDownsampleFieldProducer[]::new); - this.downsampleBucketBuilder = new DownsampleBucketBuilder(fieldProducers); + this.downsampleBucketBuilder = new DownsampleBucketBuilder(fieldProducers, dimensions); } @Override @@ -482,9 +486,11 @@ private class DownsampleBucketBuilder { private int docCount; private final AbstractDownsampleFieldProducer[] fieldProducers; private final DownsampleFieldSerializer[] groupedProducers; + private final String[] dimensions; - DownsampleBucketBuilder(AbstractDownsampleFieldProducer[] fieldProducers) { + DownsampleBucketBuilder(AbstractDownsampleFieldProducer[] fieldProducers, String[] dimensions) { this.fieldProducers = fieldProducers; + this.dimensions = dimensions; /* * The downsample field producers for aggregate_metric_double all share the same name (this is * the name they will be serialized in the target index). We group all field producers by @@ -545,12 +551,13 @@ public XContentBuilder buildDownsampleDocument() throws IOException { } builder.field(timestampField.name(), timestampFormat.format(timestamp)); builder.field(DocCountFieldMapper.NAME, docCount); + // TODO: should we make sure extracting dimension fields is backward compatible with older indices versions? // Extract dimension values from _tsid field, so we avoid loading them from doc_values - Map dimensions = (Map) DocValueFormat.TIME_SERIES_ID.format(tsid); - for (Map.Entry e : dimensions.entrySet()) { - assert e.getValue() != null; - builder.field((String) e.getKey(), e.getValue()); - } + // Map dimensions = (Map) DocValueFormat.TIME_SERIES_ID.format(tsid); + // for (Map.Entry e : dimensions.entrySet()) { + // assert e.getValue() != null; + // builder.field((String) e.getKey(), e.getValue()); + // } // Serialize fields for (DownsampleFieldSerializer fieldProducer : groupedProducers) { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java index 06e69ab4702c1..ab613c2df5529 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardPersistentTaskExecutor.java @@ -200,6 +200,7 @@ protected void doRun() throws Exception { params.downsampleConfig(), params.metrics(), params.labels(), + params.dimensions(), initialState ); downsampleShardIndexer.execute(); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardTaskParams.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardTaskParams.java index 6a4ee88a0cdef..6288fc0d6f9c7 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardTaskParams.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardTaskParams.java @@ -32,7 +32,8 @@ public record DownsampleShardTaskParams( long indexEndTimeMillis, ShardId shardId, String[] metrics, - String[] labels + String[] labels, + String[] dimensions ) implements PersistentTaskParams { public static final String NAME = DownsampleShardTask.TASK_NAME; @@ -43,6 +44,7 @@ public record DownsampleShardTaskParams( private static final ParseField SHARD_ID = new ParseField("shard_id"); private static final ParseField METRICS = new ParseField("metrics"); private static final ParseField LABELS = new ParseField("labels"); + private static final ParseField DIMENSIONS = new ParseField("dimensions"); public static final ObjectParser PARSER = new ObjectParser<>(NAME); static { @@ -57,6 +59,7 @@ public record DownsampleShardTaskParams( PARSER.declareString(DownsampleShardTaskParams.Builder::shardId, SHARD_ID); PARSER.declareStringArray(DownsampleShardTaskParams.Builder::metrics, METRICS); PARSER.declareStringArray(DownsampleShardTaskParams.Builder::labels, LABELS); + PARSER.declareStringArray(DownsampleShardTaskParams.Builder::dimensions, DIMENSIONS); } DownsampleShardTaskParams(final StreamInput in) throws IOException { @@ -67,7 +70,8 @@ public record DownsampleShardTaskParams( in.readVLong(), new ShardId(in), in.readStringArray(), - in.readStringArray() + in.readStringArray(), + in.readOptionalStringArray() ); } @@ -81,6 +85,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(SHARD_ID.getPreferredName(), shardId); builder.array(METRICS.getPreferredName(), metrics); builder.array(LABELS.getPreferredName(), labels); + builder.array(DIMENSIONS.getPreferredName(), dimensions); return builder.endObject(); } @@ -103,6 +108,7 @@ public void writeTo(StreamOutput out) throws IOException { shardId.writeTo(out); out.writeStringArray(metrics); out.writeStringArray(labels); + out.writeOptionalStringArray(dimensions); } public static DownsampleShardTaskParams fromXContent(XContentParser parser) throws IOException { @@ -123,7 +129,8 @@ public boolean equals(Object o) { && Objects.equals(shardId.id(), that.shardId.id()) && Objects.equals(shardId.getIndexName(), that.shardId.getIndexName()) && Arrays.equals(metrics, that.metrics) - && Arrays.equals(labels, that.labels); + && Arrays.equals(labels, that.labels) + && Arrays.equals(dimensions, that.dimensions); } @Override @@ -138,6 +145,7 @@ public int hashCode() { ); result = 31 * result + Arrays.hashCode(metrics); result = 31 * result + Arrays.hashCode(labels); + result = 31 * result + Arrays.hashCode(dimensions); return result; } @@ -149,6 +157,7 @@ public static class Builder { ShardId shardId; String[] metrics; String[] labels; + String[] dimensions; public Builder downsampleConfig(final DownsampleConfig downsampleConfig) { this.downsampleConfig = downsampleConfig; @@ -185,6 +194,11 @@ public Builder labels(final List labels) { return this; } + public Builder dimensions(final List dimensions) { + this.dimensions = dimensions.toArray(String[]::new); + return this; + } + public DownsampleShardTaskParams build() { return new DownsampleShardTaskParams( downsampleConfig, @@ -193,7 +207,8 @@ public DownsampleShardTaskParams build() { indexEndTimeMillis, shardId, metrics, - labels + labels, + dimensions ); } } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java index 2788932a228a8..74375bbe27939 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/FieldValueFetcher.java @@ -37,7 +37,7 @@ protected FieldValueFetcher(String name, MappedFieldType fieldType, IndexFieldDa this.name = name; this.fieldType = fieldType; this.fieldData = fieldData; - this.fieldProducer = createieldProducer(); + this.fieldProducer = createFieldProducer(); } public String name() { @@ -53,7 +53,7 @@ public AbstractDownsampleFieldProducer fieldProducer() { return fieldProducer; } - private AbstractDownsampleFieldProducer createieldProducer() { + private AbstractDownsampleFieldProducer createFieldProducer() { if (fieldType.getMetricType() != null) { return switch (fieldType.getMetricType()) { case GAUGE -> new MetricFieldProducer.GaugeMetricFieldProducer(name()); diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 322267a14d32f..4af06a1c6543f 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -334,7 +334,8 @@ protected void masterOperation( downsampleIndexName, parentTask, metricFields, - labelFields + labelFields, + dimensionFields ); } else { listener.onFailure(new ElasticsearchException("Failed to create downsample index [" + downsampleIndexName + "]")); @@ -348,7 +349,8 @@ protected void masterOperation( downsampleIndexName, parentTask, metricFields, - labelFields + labelFields, + dimensionFields ); } else { listener.onFailure(e); @@ -366,7 +368,8 @@ private void performShardDownsampling( String downsampleIndexName, TaskId parentTask, List metricFields, - List labelFields + List labelFields, + List dimensionFields ) { final int numberOfShards = sourceIndexMetadata.getNumberOfShards(); final Index sourceIndex = sourceIndexMetadata.getIndex(); @@ -386,6 +389,7 @@ private void performShardDownsampling( downsampleIndexName, metricFields, labelFields, + dimensionFields, shardId ); Predicate> predicate = runningTask -> { @@ -481,6 +485,7 @@ private static DownsampleShardTaskParams createPersistentTaskParams( final String targetIndexName, final List metricFields, final List labelFields, + final List dimensionFields, final ShardId shardId ) { return new DownsampleShardTaskParams( @@ -490,7 +495,8 @@ private static DownsampleShardTaskParams createPersistentTaskParams( parseTimestamp(sourceIndexMetadata, IndexSettings.TIME_SERIES_END_TIME), shardId, metricFields.toArray(new String[0]), - labelFields.toArray(new String[0]) + labelFields.toArray(new String[0]), + dimensionFields.toArray(new String[0]) ); } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java index a7c34cacae5be..24d1df638f80b 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleIndexerAction.java @@ -144,6 +144,7 @@ protected DownsampleIndexerAction.ShardDownsampleResponse shardOperation( request.getRollupConfig(), request.getMetricFields(), request.getLabelFields(), + request.getDimensionFields(), new DownsampleShardPersistentTaskState(DownsampleShardIndexerStatus.INITIALIZED, null) ); return indexer.execute(); diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index c0abab1234133..8beca42bf2a5f 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -627,6 +627,7 @@ public void testCancelDownsampleIndexer() throws IOException { config, new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, new String[] {}, + new String[] { FIELD_DIMENSION_1, FIELD_DIMENSION_2 }, new DownsampleShardPersistentTaskState(DownsampleShardIndexerStatus.INITIALIZED, null) ); @@ -675,6 +676,7 @@ public void testDownsampleBulkFailed() throws IOException { config, new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, new String[] {}, + new String[] { FIELD_DIMENSION_1, FIELD_DIMENSION_2 }, new DownsampleShardPersistentTaskState(DownsampleShardIndexerStatus.INITIALIZED, null) ); @@ -741,6 +743,7 @@ public void testTooManyBytesInFlight() throws IOException { config, new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, new String[] {}, + new String[] { FIELD_DIMENSION_1, FIELD_DIMENSION_2 }, new DownsampleShardPersistentTaskState(DownsampleShardIndexerStatus.INITIALIZED, null) ); /* @@ -792,6 +795,7 @@ public void testDownsampleStats() throws IOException { config, new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, new String[] {}, + new String[] { FIELD_DIMENSION_1, FIELD_DIMENSION_2 }, new DownsampleShardPersistentTaskState(DownsampleShardIndexerStatus.INITIALIZED, null) ); @@ -848,6 +852,7 @@ public void testResumeDownsample() throws IOException { config, new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, new String[] {}, + new String[] { FIELD_DIMENSION_1, FIELD_DIMENSION_2 }, new DownsampleShardPersistentTaskState( DownsampleShardIndexerStatus.STARTED, new BytesRef( @@ -922,37 +927,56 @@ public void testResumeDownsamplePartial() throws IOException { config, new String[] { FIELD_NUMERIC_1, FIELD_NUMERIC_2 }, new String[] {}, + new String[] { FIELD_DIMENSION_1 }, new DownsampleShardPersistentTaskState( DownsampleShardIndexerStatus.STARTED, + // NOTE: there is just one dimension with two possible values, this needs to be one of the two possible tsid values. new BytesRef( new byte[] { - 0x01, - 0x0C, - 0x64, - 0x69, - 0x6d, - 0x65, - 0x6E, - 0x73, - 0x69, - 0x6F, - 0x6E, - 0x5F, - 0x6B, - 0x77, + 0x24, + 0x42, + (byte) 0xe4, + (byte) 0x9f, + (byte) 0xe2, + (byte) 0xde, + (byte) 0xbb, + (byte) 0xf8, + (byte) 0xfc, + 0x7d, + 0x1a, + (byte) 0xb1, + 0x27, + (byte) 0x85, + (byte) 0xc2, + (byte) 0x8e, + 0x3a, + (byte) 0xae, + 0x38, + 0x6c, + (byte) 0xf6, + (byte) 0xae, + 0x0f, + 0x4f, + 0x44, + (byte) 0xf1, 0x73, - 0x04, - 0x64, - 0x69, - 0x6D, - 0x32 } + 0x02, + (byte) 0x90, + 0x1d, + 0x79, + (byte) 0xf8, + 0x0d, + (byte) 0xc2, + 0x7e, + (byte) 0x91, + 0x15 } ) ) ); final DownsampleIndexerAction.ShardDownsampleResponse response2 = indexer.execute(); int dim2DocCount = client().prepareSearch(sourceIndex) - .setQuery(new TermQueryBuilder(FIELD_DIMENSION_1, "dim2")) + .setQuery(new TermQueryBuilder(FIELD_DIMENSION_1, "dim1")) .setSize(10_000) .get() .getHits() diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index 3b04c276d584e..b9e073c3d70dd 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -803,7 +803,7 @@ private void assertGeoLine_TSDB( ArrayList fields = new ArrayList<>( Arrays.asList( new SortedDocValuesField("group_id", new BytesRef(testData.groups[g])), - new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.withoutHash().toBytesRef()) + new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.withHash().toBytesRef()) ) ); GeoPoint point = points.get(i); From b08c6df56b03b20e53b6d6027d192114ee7ba651 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 22 Nov 2023 20:06:58 +0100 Subject: [PATCH 028/125] fix: a few test failing because of id changes --- .../test/tsdb/25_id_generation.yml | 44 +++++++++---------- .../rest-api-spec/test/tsdb/30_snapshot.yml | 2 +- .../rest-api-spec/test/tsdb/40_search.yml | 10 ++--- .../test/tsdb/60_add_dimensions.yml | 8 ++-- .../test/tsdb/80_index_resize.yml | 2 +- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 76cf549854a01..463e107b2d407 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -76,7 +76,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs-xII2fZweptAAABeRnS7fM} + - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} - do: bulk: @@ -85,7 +85,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs-xII2fZweptAAABeRnS7fM} + - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} - do: search: @@ -97,39 +97,39 @@ generates a consistent id: - match: {hits.total.value: 9} - - match: { hits.hits.0._id: cn4exSPK93Q9eJj8AAABeRnRFAY } + - match: { hits.hits.0._id: cn4excfoxSs_KdA5AAABeRnRFAY } - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z } - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.1._id: cZZNs-xII2fZweptAAABeRnRGTM } + - match: { hits.hits.1._id: cZZNs7B9sSWsyrL5AAABeRnRGTM } - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:04.467Z } - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.2._id: cn4exSPK93Q9eJj8AAABeRnRYiY } + - match: { hits.hits.2._id: cn4excfoxSs_KdA5AAABeRnRYiY } - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:23.142Z } - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.3._id: cZZNs-xII2fZweptAAABeRnRZ1M } + - match: { hits.hits.3._id: cZZNs7B9sSWsyrL5AAABeRnRZ1M } - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:24.467Z } - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.4._id: cZZNs-xII2fZweptAAABeRnRtXM } + - match: { hits.hits.4._id: cZZNs7B9sSWsyrL5AAABeRnRtXM } - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:44.467Z } - match: { hits.hits.4._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.5._id: cn4exSPK93Q9eJj8AAABeRnR11Y } + - match: { hits.hits.5._id: cn4excfoxSs_KdA5AAABeRnR11Y } - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:50:53.142Z } - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.6._id: cn4exSPK93Q9eJj8AAABeRnR_mY } + - match: { hits.hits.6._id: cn4excfoxSs_KdA5AAABeRnR_mY } - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:03.142Z } - match: { hits.hits.6._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.7._id: cZZNs-xII2fZweptAAABeRnSA5M } + - match: { hits.hits.7._id: cZZNs7B9sSWsyrL5AAABeRnSA5M } - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:51:04.467Z } - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.8._id: cZZNs-xII2fZweptAAABeRnS7fM } + - match: { hits.hits.8._id: cZZNs7B9sSWsyrL5AAABeRnS7fM } - match: { hits.hits.8._source.@timestamp: 2021-04-28T18:52:04.467Z } - match: { hits.hits.8._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } @@ -239,7 +239,7 @@ create operation on top of old document fails: reason: id generation changed in 8.2 - do: - catch: "/\\[cn4exSPK93Q9eJj8AAABeRnR_mY\\]\\[\\{.+\\}\\@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" + catch: "/\\[cn4excfoxSs_KdA5AAABeRnR_mY\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" index: refresh: true index: test @@ -284,13 +284,13 @@ ids query: - field: k8s.pod.network.tx query: ids: - values: ["cn4exSPK93Q9eJj8AAABeRnR_mY", "cZZNs-xII2fZweptAAABeRnSA5M"] + values: ["cn4excfoxSs_KdA5AAABeRnR_mY", "cn4excfoxSs_KdA5AAABeRnR11Y"] sort: ["@timestamp"] - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "cn4exSPK93Q9eJj8AAABeRnR_mY"} - - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434595272]} - - match: {hits.hits.1._id: "cZZNs-xII2fZweptAAABeRnSA5M"} - - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [2012916202]} + - match: {hits.hits.0._id: "cn4excfoxSs_KdA5AAABeRnR11Y"} + - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434587694]} + - match: {hits.hits.1._id: "cn4excfoxSs_KdA5AAABeRnR_mY"} + - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [1434595272]} --- get: @@ -391,20 +391,20 @@ delete over _bulk: mget: index: test body: - ids: [ cn4exSPK93Q9eJj8AAABeRnR_mY, cZZNs-xII2fZweptAAABeRnSA5M ] + ids: [ cn4excfoxSs_KdA5AAABeRnR_mY, cn4excfoxSs_KdA5AAABeRnR11Y ] - match: { docs.0._index: "test" } - - match: { docs.0._id: "cn4exSPK93Q9eJj8AAABeRnR_mY" } + - match: { docs.0._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } - match: { docs.0.found: true } - match: { docs.1._index: "test" } - - match: { docs.1._id: "cZZNs-xII2fZweptAAABeRnSA5M" } + - match: { docs.1._id: "cn4excfoxSs_KdA5AAABeRnR11Y" } - match: { docs.1.found: true } - do: bulk: index: test body: - - '{"delete": {"_id": "cn4exSPK93Q9eJj8AAABeRnR_mY"}}' - - '{"delete": {"_id": "cZZNs-xII2fZweptAAABeRnSA5M"}}' + - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR_mY"}}' + - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR11Y"}}' - '{"delete": {"_id": "not found ++ not found"}}' - match: {items.0.delete.result: deleted} - match: {items.1.delete.result: deleted} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index d150a73ed7e01..e092569a5f73f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -148,5 +148,5 @@ teardown: - match: {hits.total.value: 1} - match: {hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507} - - match: {hits.hits.0.fields._tsid: [{ _tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" }]} + - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index b6678582b08ab..d0b9a3a1ceb6e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -185,7 +185,7 @@ fetch a tag: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}]} + - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} --- aggregate a dimension: @@ -319,7 +319,7 @@ sort by tsid: sort: [ "_tsid", "@timestamp" ] - match: {hits.total.value: 8} - - match: {hits.hits.0.sort: [{ _tsid : "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}, 1619635803142]} - - match: {hits.hits.1.sort: [{ _tsid : "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}, 1619635823142]} - - match: {hits.hits.4.sort: [{ _tsid : "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}, 1619635804467]} - - match: {hits.hits.7.sort: [{ _tsid : "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}, 1619635864467]} + - match: {hits.hits.0.sort: ["KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQAAAAAAAAA", 1619635803142]} + - match: {hits.hits.1.sort: ["KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQAAAAAAAAA", 1619635823142]} + - match: {hits.hits.4.sort: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0oAAAAAAAAA", 1619635804467]} + - match: {hits.hits.7.sort: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0oAAAAAAAAA", 1619635864467]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 2bf6c00874ddc..7d30f915618d5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -102,7 +102,7 @@ add dimensions to no dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "27hcKTYkXCOfYiuRWRTWygAAAAAAAAAAAAAAAAAAAAAGlT_5e6_NYGOZWULpmMG9IAlZlA" } ] } + - match: {hits.hits.0.fields._tsid: [ "JNu4XCk2JFwjn2IrkVkU1soGlT_5e6_NYGOZWULpmMG9IAlZlA" ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -150,7 +150,7 @@ add dimensions to no dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "27hcKTYkXCOfYiuRWRTWygAAAAAAAAAAAAAAAAAAAAAGlT_5e6_NYGOZWULpmMG9IAlZlA" } ] } + - match: {hits.hits.0.fields._tsid: [ "JNu4XCk2JFwjn2IrkVkU1soGlT_5e6_NYGOZWULpmMG9IAlZlA" ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -202,7 +202,7 @@ add dimensions to some dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "-CXfM53tOdTdJn8GSJ1F1QAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y"} ] } + - match: {hits.hits.0.fields._tsid: ["KPgl3zOd7TnU3SZ_BkidRdUGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y"] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -253,5 +253,5 @@ add dimensions to some dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "-CXfM53tOdTdJn8GSJ1F1QAAAAAAAAAAAAAAAAAAAAAGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y" } ] } + - match: {hits.hits.0.fields._tsid: ["KPgl3zOd7TnU3SZ_BkidRdUGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y"] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index 0ed6573c9ea19..961d0901a79f1 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -148,4 +148,4 @@ clone: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}]} + - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} From 2822d0f751ff63bfc5c7d17a6f746af065b3596f Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 23 Nov 2023 10:38:16 +0100 Subject: [PATCH 029/125] fix: some tsid values after introducing tsid hashing --- .../test/tsdb/25_id_generation.yml | 4 +- .../rest-api-spec/test/tsdb/30_snapshot.yml | 4 +- .../rest-api-spec/test/tsdb/40_search.yml | 4 +- .../test/tsdb/60_add_dimensions.yml | 4 +- .../test/tsdb/80_index_resize.yml | 4 +- .../mapper/TimeSeriesIdFieldMapperTests.java | 3 +- ...meSeriesIdFieldMapperTsidHashingTests.java | 7 ++- .../rate/TimeSeriesRateAggregatorTests.java | 14 +++--- .../test/spatial/120_position_geo_line.yml | 50 +++++++++---------- 9 files changed, 50 insertions(+), 44 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 463e107b2d407..17832607796bd 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 8.1.99,8.7.00 - 8.9.99" - reason: "tsdb indexing changed in 8.2.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + version: " - 8.1.99,8.7.00 - 8.11.99" + reason: "Introduce TSID hashing" - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index e092569a5f73f..85512e8b53b03 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: "8.7.00 - 8.9.99" - reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + version: "8.7.00 - 8.11.99" + reason: "Introduce TSID hashing" - do: snapshot.create_repository: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index d0b9a3a1ceb6e..b2646e75bf962 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 8.1.99,8.7.00 - 8.9.99" - reason: "tsdb indexing changed in 8.2.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + version: " - 8.1.99,8.7.00 - 8.11.99" + reason: "Introduce TSID hashing" - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 7d30f915618d5..7f29210df7abb 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: "8.7.00 - 8.9.99" - reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + version: "8.7.00 - 8.11.99" + reason: "Introduce TSID hashing" --- add dimensions with put_mapping: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index 961d0901a79f1..b2f1149c65e79 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 8.1.99,8.7.00 - 8.9.99" - reason: "tsdb indexing changed in 8.2.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + version: " - 8.1.99,8.7.00 - 8.11.99" + reason: "Introduce TSID hashing" features: "arbitrary_key" # Force allocating all shards to a single node so that we can shrink later. diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 6ec638a6039b1..a6e0a7b28936e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -470,7 +470,8 @@ public void testVeryLarge() throws IOException { Object tsid = TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); assertEquals( tsid, - "AWJzA2ZvbwJkMHPwBm1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd29yZHMgbWFueSB3b3JkcyA" + "AWJzA2ZvbwJkMHPwBm1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd" + + "29yZHMgbWFueSB3b3JkcyA" ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java index 66af56e67460c..502a8b5598c28 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java @@ -487,7 +487,12 @@ public void testVeryLarge() throws IOException { }); assertEquals( TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - "QZ1K9tk9UIcpA826eU6H-QAAAAAAAAAAAAAAAAAAAACv9houWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk8VBv2rLPN9GYxF8e8uXojI" + "QZ1K9tk9UIcpA826eU6H-QAAAAAAAAAAAAAAAAAAAACv9houWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWK" + + "lBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ip" + + "QZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqU" + + "GTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlB" + + "k1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk8VBv2" + + "rLPN9GYxF8e8uXojI" ); } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index 333fc888a8f0c..01440cf2a44f2 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -55,11 +55,13 @@ public void testSimple() throws IOException { Consumer verifier = r -> { assertThat(r.getBuckets(), hasSize(2)); assertThat( - ((Rate) r.getBucketByKey("{dim=1}").getAggregations().asList().get(0)).getValue(), + ((Rate) r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2LDvTu5MAoxgghmvEUd5a_i3Y3TA").getAggregations().asList().get(0)) + .getValue(), closeTo(59.0 / 3000.0 * MILLIS_IN_SECOND, 0.00001) ); assertThat( - ((Rate) r.getBucketByKey("{dim=2}").getAggregations().asList().get(0)).getValue(), + ((Rate) r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2hRiY7lpxc5PgA4k16syGbZGgr0g").getAggregations().asList().get(0)) + .getValue(), closeTo(206.0 / 4000.0 * MILLIS_IN_SECOND, 0.00001) ); }; @@ -83,12 +85,10 @@ public void testNestedWithinDateHistogram() throws IOException { Consumer verifier = r -> { assertThat(r.getBuckets(), hasSize(2)); assertThat( - r.getBucketByKey("NFFUy14C9UcX3MnFnsFrpf0AAAAAAAAAAAAAAAAAAAAAiw707uTAKMYIIZrxFHeWv4t2N0w"), + r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2LDvTu5MAoxgghmvEUd5a_i3Y3TA"), instanceOf(InternalTimeSeries.InternalBucket.class) ); - InternalDateHistogram hb = r.getBucketByKey("NFFUy14C9UcX3MnFnsFrpf0AAAAAAAAAAAAAAAAAAAAAiw707uTAKMYIIZrxFHeWv4t2N0w") - .getAggregations() - .get("date"); + InternalDateHistogram hb = r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2LDvTu5MAoxgghmvEUd5a_i3Y3TA").getAggregations().get("date"); { Rate rate = hb.getBuckets().get(1).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((60 - 37 + 14) / 2000.0 * MILLIS_IN_SECOND, 0.00001)); @@ -97,7 +97,7 @@ public void testNestedWithinDateHistogram() throws IOException { Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((37 - 15) / 1000.0 * MILLIS_IN_SECOND, 0.00001)); } - hb = r.getBucketByKey("NFFUy14C9UcX3MnFnsFrpf0AAAAAAAAAAAAAAAAAAAAAoUYmO5acXOT4AOJNerMhm2RoK9I").getAggregations().get("date"); + hb = r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2hRiY7lpxc5PgA4k16syGbZGgr0g").getAggregations().get("date"); { Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((150 - 74) / 1000.0 * MILLIS_IN_SECOND, 0.00001)); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml index 580505ba89bc8..240fa4d17c17f 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 8.7.99" - reason: position metric introduced in 8.8.0 + version: " - 8.11.99" + reason: "Introduce TSID hashing" - do: indices.create: index: locations @@ -87,8 +87,8 @@ setup: - length: { aggregations.by_time_series.buckets: 3 } - match: aggregations.by_time_series.buckets: - "{city=Paris}": - key: { city: Paris } + "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": + key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" doc_count: 2 museum_tour: type: Feature @@ -96,22 +96,22 @@ setup: coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] type: LineString properties: { complete: true } - "{city=Antwerp}": - key: { city: Antwerp } - doc_count: 3 + "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": + key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" + doc_count: 5 museum_tour: type: Feature geometry: - coordinates: [ [ 4.401384, 51.220292 ], [ 4.405819, 51.221758 ], [ 4.4052, 51.2229 ] ] + coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] type: LineString properties: { complete: true } - "{city=Amsterdam}": - key: { city: Amsterdam } - doc_count: 5 + "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": + key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" + doc_count: 3 museum_tour: type: Feature geometry: - coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] + coordinates: [ [ 4.401384, 51.220292 ], [ 4.405819, 51.221758 ], [ 4.4052, 51.2229 ] ] type: LineString properties: { complete: true } @@ -140,8 +140,8 @@ setup: - length: { aggregations.by_time_series.buckets: 3 } - match: aggregations.by_time_series.buckets: - "{city=Paris}": - key: { city: Paris } + "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": + key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" doc_count: 2 museum_tour: type: Feature @@ -149,17 +149,8 @@ setup: coordinates: [ [ 2.327, 48.86 ], [ 2.336389, 48.861111 ] ] type: LineString properties: { complete: true } - "{city=Antwerp}": - key: { city: Antwerp } - doc_count: 3 - museum_tour: - type: Feature - geometry: - coordinates: [ [ 4.4052, 51.2229 ], [ 4.405819, 51.221758 ], [ 4.401384, 51.220292 ] ] - type: LineString - properties: { complete: true } - "{city=Amsterdam}": - key: { city: Amsterdam } + "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": + key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" doc_count: 5 museum_tour: type: Feature @@ -171,6 +162,15 @@ setup: [ 4.889187, 52.373184 ] ] type: LineString properties: { complete: true } + "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": + key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" + doc_count: 3 + museum_tour: + type: Feature + geometry: + coordinates: [ [ 4.4052, 51.2229 ], [ 4.405819, 51.221758 ], [ 4.401384, 51.220292 ] ] + type: LineString + properties: { complete: true } --- "geo_line on position field time time-series with invalid sort specified": From 6ce797fc2c128c0937a61ff552f11eb5f9305850 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 23 Nov 2023 12:58:49 +0100 Subject: [PATCH 030/125] fix: some more tests --- ...riesAggregationsUnlimitedDimensionsIT.java | 19 +--- .../TimeSeriesNestedAggregationsIT.java | 23 +---- .../elasticsearch/search/DocValueFormat.java | 13 +-- .../bucket/composite/InternalComposite.java | 29 ++++++- .../rest-api-spec/test/70_time_series.yml | 51 +++++------ .../test/security/authz/70_tsdb.yml | 6 +- .../test/spatial/120_position_geo_line.yml | 86 +++++++++---------- 7 files changed, 112 insertions(+), 115 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java index d8e38a9affea3..c4dc4c4f36c30 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; @@ -36,7 +35,6 @@ import java.io.IOException; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.function.Supplier; @@ -196,23 +194,14 @@ public void testCardinalityByTsid() { } private static void assertTimeSeriesAggregation(final InternalTimeSeries timeSeriesAggregation) { - final List> dimensions = timeSeriesAggregation.getBuckets() - .stream() - .map(InternalTimeSeries.InternalBucket::getKey) - .toList(); + final List dimensions = timeSeriesAggregation.getBuckets().stream().map(InternalTimeSeries.InternalBucket::getKey).toList(); // NOTE: only two time series expected as a result of having just two distinct values for the last dimension assertEquals(2, dimensions.size()); - final Map firstTimeSeries = dimensions.get(0); - final Map secondTimeSeries = dimensions.get(1); + final Object firstTimeSeries = dimensions.get(0); + final Object secondTimeSeries = dimensions.get(1); - assertTsid(firstTimeSeries); - assertTsid(secondTimeSeries); - } - - private static void assertTsid(final Map timeSeries) { - final Map.Entry tsidEntry = timeSeries.entrySet().stream().toList().get(0); - assertEquals(TimeSeriesIdFieldMapper.NAME, tsidEntry.getKey()); + assertNotEquals(firstTimeSeries, secondTimeSeries); } private static void assertCardinality(final InternalCardinality cardinalityAggregation, int expectedCardinality) { diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java index 7fddc65ac3e03..566d083d75760 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java @@ -29,14 +29,12 @@ import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; -import org.hamcrest.Matchers; import org.junit.Before; import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.function.Supplier; @@ -209,27 +207,14 @@ public void testCardinalityByTsid() { } private static void assertTimeSeriesAggregation(final InternalTimeSeries timeSeriesAggregation) { - final List> dimensions = timeSeriesAggregation.getBuckets() - .stream() - .map(InternalTimeSeries.InternalBucket::getKey) - .toList(); + final List dimensions = timeSeriesAggregation.getBuckets().stream().map(InternalTimeSeries.InternalBucket::getKey).toList(); // NOTE: only two time series expected as a result of having just two distinct values for the last dimension assertEquals(2, dimensions.size()); - final Map firstTimeSeries = dimensions.get(0); - final Map secondTimeSeries = dimensions.get(1); + final Object firstTimeSeries = dimensions.get(0); + final Object secondTimeSeries = dimensions.get(1); - assertTsid(firstTimeSeries); - assertTsid(secondTimeSeries); - } - - private static void assertTsid(final Map timeSeries) { - timeSeries.entrySet().stream().sorted(Map.Entry.comparingByKey()).limit(numberOfDimensions - 2).forEach(entry -> { - assertThat(entry.getValue().toString(), Matchers.equalTo(FOO_DIM_VALUE)); - }); - timeSeries.entrySet().stream().sorted(Map.Entry.comparingByKey()).skip(numberOfDimensions - 1).forEach(entry -> { - assertThat(entry.getValue().toString(), Matchers.oneOf(BAR_DIM_VALUE, BAZ_DIM_VALUE)); - }); + assertNotEquals(firstTimeSeries, secondTimeSeries); } private static void assertCardinality(final InternalCardinality cardinalityAggregation, int expectedCardinality) { diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 1ff067f24f29b..2fa28d555fc60 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -697,10 +697,13 @@ public Object format(BytesRef value) { @Override public BytesRef parseBytesRef(Object value) { - if (value instanceof BytesRef) { - return (BytesRef) value; + if (value instanceof BytesRef valueAsBytesRef) { + return new BytesRef(Base64.getUrlEncoder().withoutPadding().encodeToString(valueAsBytesRef.bytes)); } - return plainTsidParseBytesRef(value); + if (value instanceof String valueAsString) { + return new BytesRef(valueAsString); + } + return parseBytesRefMap(value); } /** @@ -708,9 +711,9 @@ public BytesRef parseBytesRef(Object value) { * Tsid hashing does not allow us to parse the tsid extracting dimension fields key/values pairs. * @param value The Map encoding tsid dimension fields key/value pairs. * - * @return + * @return a {@link BytesRef} representing a map of key/value pairs */ - private BytesRef plainTsidParseBytesRef(Object value) { + private BytesRef parseBytesRefMap(Object value) { if (value instanceof Map == false) { throw new IllegalArgumentException("Cannot parse tsid object [" + value + "]"); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java index f2c601e412f92..3233b6e276f7b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java @@ -13,6 +13,7 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.Aggregations; @@ -519,14 +520,38 @@ static Object formatObject(Object obj, DocValueFormat format) { } Object formatted = obj; Object parsed; - if (obj.getClass() == BytesRef.class) { + if (obj.getClass() == BytesRef.class && format == DocValueFormat.TIME_SERIES_ID) { + BytesRef value = (BytesRef) obj; + formatted = format.format(value); + parsed = format.parseBytesRef(formatted); + // NOTE: we cannot parse the Base64 encoding representation of the tsid and get back the original BytesRef + String objBase64 = TimeSeriesIdFieldMapper.decodeTsid(value).toString(); + BytesRef objBase64BytesRef = new BytesRef(objBase64); + if (objBase64BytesRef.equals(parsed) == false) { + throw new IllegalArgumentException( + "Format [" + + format + + "] created output it couldn't parse for value [" + + obj + + "] " + + "of type [" + + obj.getClass() + + "]. formatted value: [" + + formatted + + "(" + + parsed.getClass() + + ")]" + ); + } + } + if (obj.getClass() == BytesRef.class && format != DocValueFormat.TIME_SERIES_ID) { BytesRef value = (BytesRef) obj; if (format == DocValueFormat.RAW) { formatted = value.utf8ToString(); } else { formatted = format.format(value); } - parsed = format.parseBytesRef(formatted); + parsed = format.parseBytesRef(formatted.toString()); if (parsed.equals(obj) == false) { throw new IllegalArgumentException( "Format [" diff --git a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml index 150c90faf175a..5d4a971c6fd1b 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml +++ b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml @@ -64,10 +64,10 @@ fetch the _tsid: sort: [ _tsid ] - match: {hits.total.value: 2} - - match: {hits.hits.0.fields._tsid: [{metricset: aa, ul: 9223372036854775807}]} + - match: {hits.hits.0.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} - match: {hits.hits.0.fields.metricset: [aa]} - match: {hits.hits.0.fields.ul: [9223372036854775807]} - - match: {hits.hits.1.fields._tsid: [{metricset: aa, ul: 9223372036854775808}]} + - match: {hits.hits.1.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} - match: {hits.hits.1.fields.metricset: [aa]} - match: {hits.hits.1.fields.ul: [9223372036854775808]} @@ -119,14 +119,14 @@ aggregate the _tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {metricset: aa, ul: 9223372036854775807}} - - match: {aggregations.tsids.buckets.0.doc_count: 3} - - match: {aggregations.tsids.buckets.1.key: {metricset: aa, ul: 9223372036854775808}} + - match: {aggregations.tsids.buckets.0.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62VWzfUKafNxTRdD8v6_Z8PbKJ6TQ"} + - match: {aggregations.tsids.buckets.0.doc_count: 1} + - match: {aggregations.tsids.buckets.1.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"} - match: {aggregations.tsids.buckets.1.doc_count: 3} - - match: {aggregations.tsids.buckets.2.key: {metricset: aa, ul: 18446744073709551614}} + - match: {aggregations.tsids.buckets.2.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62erJQlP1_gz9tjb7S743_9tpXyfM"} - match: {aggregations.tsids.buckets.2.doc_count: 1} - - match: {aggregations.tsids.buckets.3.key: {metricset: aa, ul: 18446744073709551615}} - - match: {aggregations.tsids.buckets.3.doc_count: 1 } + - match: {aggregations.tsids.buckets.3.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"} + - match: {aggregations.tsids.buckets.3.doc_count: 3} --- @@ -147,14 +147,14 @@ sort by tsid: - match: {hits.total.value: 8 } - - match: {hits.hits.0.fields._tsid: [{metricset: aa, ul: 9223372036854775807}]} - - match: {hits.hits.1.fields._tsid: [{metricset: aa, ul: 9223372036854775807}]} - - match: {hits.hits.2.fields._tsid: [{metricset: aa, ul: 9223372036854775807}]} - - match: {hits.hits.3.fields._tsid: [{metricset: aa, ul: 9223372036854775808}]} - - match: {hits.hits.4.fields._tsid: [{metricset: aa, ul: 9223372036854775808}]} - - match: {hits.hits.5.fields._tsid: [{metricset: aa, ul: 9223372036854775808}]} - - match: {hits.hits.6.fields._tsid: [{metricset: aa, ul: 18446744073709551614}]} - - match: {hits.hits.7.fields._tsid: [{metricset: aa, ul: 18446744073709551615}]} + - match: {hits.hits.0.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62VWzfUKafNxTRdD8v6_Z8PbKJ6TQ"]} + - match: {hits.hits.1.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} + - match: {hits.hits.2.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} + - match: {hits.hits.3.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} + - match: {hits.hits.4.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62erJQlP1_gz9tjb7S743_9tpXyfM"]} + - match: {hits.hits.5.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} + - match: {hits.hits.6.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} + - match: {hits.hits.7.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} --- composite aggregation on tsid: @@ -180,18 +180,13 @@ composite aggregation on tsid: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key.tsid.ul: 9223372036854775807 } - - match: { aggregations.tsids.buckets.0.key.tsid.metricset: "aa" } - - match: { aggregations.tsids.buckets.0.doc_count: 3 } - - match: { aggregations.tsids.buckets.1.key.tsid.ul: 9223372036854775808 } - - match: { aggregations.tsids.buckets.1.key.tsid.metricset: "aa" } + - match: { aggregations.tsids.buckets.0.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62VWzfUKafNxTRdD8v6_Z8PbKJ6TQ" } + - match: { aggregations.tsids.buckets.0.doc_count: 1 } + - match: { aggregations.tsids.buckets.1.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA" } - match: { aggregations.tsids.buckets.1.doc_count: 3 } - - match: { aggregations.tsids.buckets.2.key.tsid.ul: 18446744073709551614 } - - match: { aggregations.tsids.buckets.2.key.tsid.metricset: "aa" } + - match: { aggregations.tsids.buckets.2.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62erJQlP1_gz9tjb7S743_9tpXyfM" } - match: { aggregations.tsids.buckets.2.doc_count: 1 } - - match: { aggregations.tsids.buckets.3.key.tsid.ul: 18446744073709551615 } - - match: { aggregations.tsids.buckets.3.key.tsid.metricset: "aa" } - - match: { aggregations.tsids.buckets.3.doc_count: 1 } + - match: { aggregations.tsids.buckets.3.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U" } + - match: { aggregations.tsids.buckets.3.doc_count: 3 } - - match: { aggregations.tsids.after_key.tsid.ul: 18446744073709551615 } - - match: { aggregations.tsids.after_key.tsid.metricset: "aa" } + - match: { aggregations.tsids.after_key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml index 8044e9bc3b8ab..66ce482f70603 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml @@ -104,13 +104,13 @@ document level security on tag: - match: {hits.total.value: 4} - length: {aggregations.tsids.buckets: 1} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.0.doc_count: 4} --- document level security on dimension: - skip: - version: " - 8.0.99" + version: " - 8.11.99" reason: _tsid support introduced in 8.1.0 features: headers @@ -151,7 +151,7 @@ document level security on dimension: - match: { hits.total.value: 4 } - length: { aggregations.tsids.buckets: 1 } - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.0.doc_count: 4} --- diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml index 240fa4d17c17f..5ba221faf3c8b 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml @@ -216,8 +216,8 @@ setup: - length: { aggregations.by_time_series.buckets: 3 } - match: aggregations.by_time_series.buckets: - "{city=Paris}": - key: { city: Paris } + "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": + key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" doc_count: 2 museum_tour: type: Feature @@ -225,22 +225,22 @@ setup: coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] type: LineString properties: { complete: true } - "{city=Antwerp}": - key: { city: Antwerp } - doc_count: 3 + "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": + key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" + doc_count: 5 museum_tour: type: Feature geometry: - coordinates: [ [ 4.401384, 51.220292 ], [ 4.405819, 51.221758 ], [ 4.4052, 51.2229 ] ] + coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] type: LineString properties: { complete: true } - "{city=Amsterdam}": - key: { city: Amsterdam } - doc_count: 5 + "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": + key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" + doc_count: 3 museum_tour: type: Feature geometry: - coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] + coordinates: [ [ 4.401384, 51.220292 ], [ 4.405819, 51.221758 ], [ 4.4052, 51.2229 ] ] type: LineString properties: { complete: true } @@ -326,40 +326,40 @@ setup: - length: { aggregations.with_time_series.buckets: 3 } - match: aggregations.with_time_series.buckets: - "{city=Amsterdam}": - key: { city: Amsterdam } - doc_count: 5 + "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": + key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" + doc_count: 2 museums: doc_count_error_upper_bound: 0 sum_other_doc_count: 0 buckets: - - key: Amsterdam - doc_count: 5 + - key: Paris + doc_count: 2 museum_tour: type: Feature geometry: - coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] + coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] type: LineString properties: complete: true - "{city=Paris}": - key: { city: Paris } - doc_count: 2 + "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": + key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" + doc_count: 5 museums: doc_count_error_upper_bound: 0 sum_other_doc_count: 0 buckets: - - key: Paris - doc_count: 2 + - key: Amsterdam + doc_count: 5 museum_tour: type: Feature geometry: - coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] + coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] type: LineString properties: complete: true - "{city=Antwerp}": - key: { city: Antwerp } + "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": + key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" doc_count: 3 museums: doc_count_error_upper_bound: 0 @@ -447,49 +447,49 @@ setup: - length: { aggregations.with_time_series.buckets: 3 } - match: aggregations.with_time_series.buckets: - "{city=Amsterdam}": - key: { city: Amsterdam } - doc_count: 5 + "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": + key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" + doc_count: 2 museums: doc_count_error_upper_bound: 0 sum_other_doc_count: 0 buckets: - key: Museum - doc_count: 3 - museum_tour: - type: Feature - geometry: - coordinates: [ [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] - type: LineString - properties: - complete: true - - key: Attraction doc_count: 2 museum_tour: type: Feature geometry: - coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ] ] + coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] type: LineString properties: complete: true - "{city=Paris}": - key: { city: Paris } - doc_count: 2 + "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": + key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" + doc_count: 5 museums: doc_count_error_upper_bound: 0 sum_other_doc_count: 0 buckets: - key: Museum + doc_count: 3 + museum_tour: + type: Feature + geometry: + coordinates: [ [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] + type: LineString + properties: + complete: true + - key: Attraction doc_count: 2 museum_tour: type: Feature geometry: - coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] + coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ] ] type: LineString properties: complete: true - "{city=Antwerp}": - key: { city: Antwerp } + "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": + key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" doc_count: 3 museums: doc_count_error_upper_bound: 0 From d25620985fda9c8cab0dd50dd093b734c489a2ef Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 23 Nov 2023 13:05:59 +0100 Subject: [PATCH 031/125] fix: update yaml skip version --- .../test/aggregations/time_series.yml | 8 +++--- .../test/tsdb/25_id_generation.yml | 10 +++---- .../rest-api-spec/test/tsdb/30_snapshot.yml | 2 +- .../rest-api-spec/test/tsdb/40_search.yml | 14 +++++----- .../test/tsdb/60_add_dimensions.yml | 22 +++++++-------- .../test/tsdb/70_dimension_types.yml | 28 +++++++++---------- .../test/tsdb/80_index_resize.yml | 2 +- .../test/spatial/120_position_geo_line.yml | 2 +- 8 files changed, 44 insertions(+), 44 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index ac5725534f06f..7f003ba31964e 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -54,8 +54,8 @@ setup: --- "Basic test": - skip: - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: @@ -107,8 +107,8 @@ setup: --- "Size test": - skip: - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 17832607796bd..e666b3145a89e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -2,7 +2,7 @@ setup: - skip: version: " - 8.1.99,8.7.00 - 8.11.99" - reason: "Introduce TSID hashing" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -258,8 +258,8 @@ create operation on top of old document fails: --- create operation on top of old document fails over bulk: - skip: - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: bulk: @@ -273,8 +273,8 @@ create operation on top of old document fails over bulk: --- ids query: - skip: - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 85512e8b53b03..142e17e4f665c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -2,7 +2,7 @@ setup: - skip: version: "8.7.00 - 8.11.99" - reason: "Introduce TSID hashing" + reason: _tsid hasing introduced in 8.12 - do: snapshot.create_repository: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index b2646e75bf962..b280e2f901a17 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -2,7 +2,7 @@ setup: - skip: version: " - 8.1.99,8.7.00 - 8.11.99" - reason: "Introduce TSID hashing" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -171,8 +171,8 @@ fetch a tag: --- "fetch the tsid": - skip: - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: @@ -266,8 +266,8 @@ aggregate a tag: --- "aggregate the tsid": - skip: - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: @@ -309,8 +309,8 @@ aggregate a tag: --- sort by tsid: - skip: - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 7f29210df7abb..adedd4588b53a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -2,13 +2,13 @@ setup: - skip: version: "8.7.00 - 8.11.99" - reason: "Introduce TSID hashing" + reason: _tsid hasing introduced in 8.12 --- add dimensions with put_mapping: - skip: - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -60,8 +60,8 @@ add dimensions with put_mapping: --- add dimensions to no dims with dynamic_template over index: - skip: - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -108,8 +108,8 @@ add dimensions to no dims with dynamic_template over index: --- add dimensions to no dims with dynamic_template over bulk: - skip: - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -156,8 +156,8 @@ add dimensions to no dims with dynamic_template over bulk: --- add dimensions to some dims with dynamic_template over index: - skip: - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -208,8 +208,8 @@ add dimensions to some dims with dynamic_template over index: --- add dimensions to some dims with dynamic_template over bulk: - skip: - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index e1c14a28e7d27..478237b6a19f1 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -8,8 +8,8 @@ setup: keyword dimension: - skip: features: close_to - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: @@ -82,8 +82,8 @@ keyword dimension: flattened dimension: - skip: features: close_to - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: @@ -186,8 +186,8 @@ flattened dimension: flattened empty dimension: - skip: features: close_to - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: @@ -267,8 +267,8 @@ flattened empty dimension: flattened field missing routing path field: - skip: features: close_to - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -380,8 +380,8 @@ flattened field missing routing path field: flattened field misspelled routing path field: - skip: features: close_to - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -472,8 +472,8 @@ flattened field misspelled routing path field: long dimension: - skip: features: close_to - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -547,8 +547,8 @@ long dimension: ip dimension: - skip: features: close_to - version: " - 8.10.99" - reason: _tsid hasing introduced in 8.11 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index b2f1149c65e79..0a83867aebf28 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -2,7 +2,7 @@ setup: - skip: version: " - 8.1.99,8.7.00 - 8.11.99" - reason: "Introduce TSID hashing" + reason: _tsid hasing introduced in 8.12 features: "arbitrary_key" # Force allocating all shards to a single node so that we can shrink later. diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml index 5ba221faf3c8b..85e1e1a5800ad 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml @@ -2,7 +2,7 @@ setup: - skip: version: " - 8.11.99" - reason: "Introduce TSID hashing" + reason: _tsid hasing introduced in 8.12 - do: indices.create: index: locations From a2b7ab3a8ca449c3d942b06b8ab5dedb3ac08d5b Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 23 Nov 2023 16:32:07 +0100 Subject: [PATCH 032/125] fix: some more tests --- .../timeseries/TimeSeriesAggregatorTests.java | 73 +++++-------------- .../rest-api-spec/test/delete/70_tsdb.yml | 12 +-- .../test/tsdb/140_routing_path.yml | 11 +-- .../test/tsdb/25_id_generation.yml | 36 ++++----- .../rest-api-spec/test/tsdb/50_alias.yml | 8 +- .../test/analytics/reset_tracking_rate.yml | 5 +- 6 files changed, 48 insertions(+), 97 deletions(-) diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index 4326806c90e5e..96c4eb0d475b1 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -66,34 +66,19 @@ public void testStandAloneTimeSeriesWithSum() throws IOException { }, ts -> { assertThat(ts.getBuckets(), hasSize(3)); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); assertThat( - ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, - equalTo(4L) - ); - assertThat( - ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o") - .getAggregations() - .get("sum")).value(), + ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").getAggregations().get("sum")).value(), equalTo(22.0) ); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); assertThat( - ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, - equalTo(2L) - ); - assertThat( - ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg") - .getAggregations() - .get("sum")).value(), + ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").getAggregations().get("sum")).value(), equalTo(6.0) ); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); assertThat( - ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, - equalTo(2L) - ); - assertThat( - ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4") - .getAggregations() - .get("sum")).value(), + ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").getAggregations().get("sum")).value(), equalTo(8.0) ); @@ -176,33 +161,24 @@ public void testMultiBucketAggregationAsSubAggregation() throws IOException { Consumer verifier = ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat( - ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, - equalTo(4L) - ); - InternalDateHistogram byTimeStampBucket = ts.getBucketByKey( - "C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o" - ).getAggregations().get("by_timestamp"); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); + InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o") + .getAggregations() + .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 4, false, null, InternalAggregations.EMPTY)) ); - assertThat( - ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, - equalTo(2L) - ); - byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg") + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); + byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg") .getAggregations() .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); - assertThat( - ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, - equalTo(2L) - ); - byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4") + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); + byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4") .getAggregations() .get("by_timestamp"); assertThat( @@ -224,24 +200,9 @@ public void testAggregationSize() throws IOException { List> verifiers = new ArrayList>(); - verifiers.add( - ts -> assertThat( - ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, - equalTo(4L) - ) - ); - verifiers.add( - ts -> assertThat( - ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, - equalTo(2L) - ) - ); - verifiers.add( - ts -> assertThat( - ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, - equalTo(2L) - ) - ); + verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L))); for (int i = 1; i <= 3; i++) { int size = i; diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml index 6f761f2d4a5e1..82e064304ce4c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml @@ -1,14 +1,8 @@ ---- -setup: - - skip: - version: "8.7.00 - 8.9.99" - reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - --- "basic tsdb delete": - skip: - version: " - 8.8.0" - reason: fixed in 8.8.1 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -49,7 +43,7 @@ setup: location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhDzuYSDy965GAAABiHD35_g } + - match: { _id: crxuhC8WO3aVdhvtAAABiHD35_g } - match: { result: created } - match: { _version: 1 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 9b7052fd80810..e1393c4213fe8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -1,10 +1,3 @@ ---- -setup: - - skip: - version: " - 8.9.99" - reason: "counter field support added in 8.10" - features: close_to - --- missing routing path field: - skip: @@ -133,8 +126,8 @@ missing dimension on routing path field: multi-value routing path field: - skip: features: close_to - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index e666b3145a89e..4e5edc6400e97 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -136,8 +136,8 @@ generates a consistent id: --- index a new document on top of an old one: - skip: - version: " - 8.1.99" - reason: indexing on top of another document support added in 8.2 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: @@ -170,7 +170,7 @@ index a new document on top of an old one: network: tx: 111434595272 rx: 430605511 - - match: {_id: cn4exSPK93Q9eJj8AAABeRnR_mY} + - match: {_id: cn4excfoxSs_KdA5AAABeRnR_mY} - do: search: @@ -215,7 +215,7 @@ index a new document on top of an old one over bulk: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: {items.0.index._id: cn4exSPK93Q9eJj8AAABeRnR_mY} + - match: {items.0.index._id: cn4excfoxSs_KdA5AAABeRnR_mY} - do: search: @@ -268,7 +268,7 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[cn4exSPK93Q9eJj8AAABeRnR_mY][{_tsid=KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ}@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[cn4excfoxSs_KdA5AAABeRnR_mY][KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQAAAAAAAAA@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: @@ -295,15 +295,15 @@ ids query: --- get: - skip: - version: " - 8.1.99" - reason: ids generation changed in 8.2 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: get: index: test - id: cZZNs-xII2fZweptAAABeRnSA5M + id: cZZNs7B9sSWsyrL5AAABeRnSA5M - match: {_index: test} - - match: {_id: cZZNs-xII2fZweptAAABeRnSA5M} + - match: {_id: cZZNs7B9sSWsyrL5AAABeRnSA5M} - match: _source: "@timestamp": "2021-04-28T18:51:04.467Z" @@ -345,13 +345,13 @@ get with routing: --- delete: - skip: - version: " - 8.1.99" - reason: ids generation changed in 8.2 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: delete: index: test - id: cn4exSPK93Q9eJj8AAABeRnRFAY + id: cn4excfoxSs_KdA5AAABeRnR_mY - match: {result: deleted} --- @@ -414,8 +414,8 @@ delete over _bulk: --- routing_path matches deep object: - skip: - version: " - 8.1.99" - reason: id generation changed in 8.2 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -454,13 +454,13 @@ routing_path matches deep object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"bar": {"baz": {"uid": "uid1"}}}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: OcEOGeykhIEIYuXSAAABeRnRGTM} + - match: {items.0.index._id: OcEOGchJrjH1fFX8AAABeRnRGTM} --- routing_path matches object: - skip: - version: " - 8.1.99" - reason: id generation changed in 8.2 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -495,4 +495,4 @@ routing_path matches object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"uid": "uid1"}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: 8bgiqXPBT_mM6CX6AAABeRnRGTM} + - match: {items.0.index._id: 8bgiqW9JKwAyp1bZAAABeRnRGTM} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index 00005d3e1881a..a36bd16647b93 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -64,8 +64,8 @@ setup: --- search an alias: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.put_alias: @@ -85,9 +85,9 @@ search an alias: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} + - match: {aggregations.tsids.buckets.0.key: "AgtrOHMucG9kLnVpZHMkOTQ3ZTRjZWQtMTc4Ni00ZTUzLTllMGMtNWM0NDdlOTU5NTA3CW1ldHJpY3NldHMDcG9k"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} + - match: {aggregations.tsids.buckets.1.key: "AgtrOHMucG9kLnVpZHMkOTQ3ZTRjZWQtMTc4Ni00ZTUzLTllMGMtNWM0NDdlOTU5NTA3CW1ldHJpY3NldHMDcG9k"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml index 62115b3b87081..e6e9a8a0aa295 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml @@ -103,6 +103,9 @@ setup: - '{ "key": "bar", "val": 20, "@timestamp": "2021-01-01T02:40:00Z" }' --- "test resets do not lead to negative rate": + - skip: + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: index: tsdb-* @@ -126,7 +129,7 @@ setup: - match: { hits.total.value: 9 } - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } + - match: { aggregations.ts.buckets.0.key: "JEMc3XgEDWzF216sBF5nxwW5AZB2-M4K_VYiHrKDjUNX3zHVkw" } - match: { aggregations.ts.buckets.0.doc_count: 9 } - gte: { aggregations.ts.buckets.0.rate.value: 0.0 } From 16f5f03048e3493615b335dcf393462908bf1bae Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 23 Nov 2023 16:44:19 +0100 Subject: [PATCH 033/125] fix: merge conflict --- .../support/TimeSeriesDimensionsLimitIT.java | 35 ++++++------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java index 09cf838fb4925..3b8f3f05d9c04 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java @@ -43,9 +43,8 @@ public void testDimensionFieldNameLimit() throws IOException { () -> List.of("routing_field"), dimensionFieldLimit ); - final Exception ex = expectThrows( - DocumentParsingException.class, - () -> prepareIndex("test").setSource( + final DocWriteResponse response = client().prepareIndex("test") + .setSource( "routing_field", randomAlphaOfLength(10), dimensionFieldName, @@ -54,12 +53,6 @@ public void testDimensionFieldNameLimit() throws IOException { randomIntBetween(10, 20), "@timestamp", Instant.now().toEpochMilli() - ).get() - ); - assertThat( - ex.getCause().getMessage(), - equalTo( - "Dimension name must be less than [512] bytes but [" + dimensionFieldName + "] was [" + dimensionFieldName.length() + "]." ) .get(); assertEquals(RestStatus.CREATED.getStatus(), response.status().getStatus()); @@ -74,20 +67,14 @@ public void testDimensionFieldValueLimit() throws IOException { dimensionFieldLimit ); long startTime = Instant.now().toEpochMilli(); - prepareIndex("test").setSource("field", randomAlphaOfLength(1024), "gauge", randomIntBetween(10, 20), "@timestamp", startTime) + final DocWriteResponse response1 = client().prepareIndex("test") + .setSource("field", randomAlphaOfLength(1024), "gauge", randomIntBetween(10, 20), "@timestamp", startTime) .get(); - final Exception ex = expectThrows( - DocumentParsingException.class, - () -> prepareIndex("test").setSource( - "field", - randomAlphaOfLength(1025), - "gauge", - randomIntBetween(10, 20), - "@timestamp", - startTime + 1 - ).get() - ); - assertThat(ex.getCause().getMessage(), equalTo("Dimension fields must be less than [1024] bytes but was [1025].")); + final DocWriteResponse response2 = client().prepareIndex("test") + .setSource("field", randomAlphaOfLength(1025), "gauge", randomIntBetween(10, 20), "@timestamp", startTime + 1) + .get(); + assertEquals(RestStatus.CREATED.getStatus(), response1.status().getStatus()); + assertEquals(RestStatus.CREATED.getStatus(), response2.status().getStatus()); } public void testTotalNumberOfDimensionFieldsLimit() { @@ -169,8 +156,8 @@ public void testTotalDimensionFieldsSizeLuceneLimitPlusOne() throws IOException for (int i = 0; i < dimensionFieldLimit; i++) { source.put(dimensionFieldNames.get(i), randomAlphaOfLength(1024)); } - final Exception ex = expectThrows(DocumentParsingException.class, () -> prepareIndex("test").setSource(source).get()); - assertEquals("_tsid longer than [32766] bytes [33903].", ex.getCause().getMessage()); + final DocWriteResponse response = client().prepareIndex("test").setSource(source).get(); + assertEquals(RestStatus.CREATED.getStatus(), response.status().getStatus()); } private void createTimeSeriesIndex( From c820e8771c007b89477151c5f7c157a5f94ba72e Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 23 Nov 2023 17:48:35 +0100 Subject: [PATCH 034/125] fix: other tsid tests --- .../TimeSeriesAggregatorTsidHashingTests.java | 73 ++++------------ .../test/tsdb/140_routing_path.yml | 22 ++--- .../test/tsdb/60_add_dimensions.yml | 8 +- .../test/tsdb/70_dimension_types.yml | 84 +++++-------------- .../test/tsdb/80_index_resize.yml | 10 +-- .../rest-api-spec/test/analytics/100_tsdb.yml | 10 +-- 6 files changed, 58 insertions(+), 149 deletions(-) diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java index 7b7c3ee4467a5..22d8deea00cc9 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java @@ -95,34 +95,19 @@ public void testStandAloneTimeSeriesWithSum() throws IOException { }, ts -> { assertThat(ts.getBuckets(), hasSize(3)); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); assertThat( - ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg}").docCount, - equalTo(2L) - ); - assertThat( - ((Sum) ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg}") - .getAggregations() - .get("sum")).value(), + ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").getAggregations().get("sum")).value(), equalTo(6.0) ); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); assertThat( - ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4}").docCount, - equalTo(2L) - ); - assertThat( - ((Sum) ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4}") - .getAggregations() - .get("sum")).value(), + ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").getAggregations().get("sum")).value(), equalTo(8.0) ); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); assertThat( - ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o}").docCount, - equalTo(4L) - ); - assertThat( - ((Sum) ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o}") - .getAggregations() - .get("sum")).value(), + ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").getAggregations().get("sum")).value(), equalTo(22.0) ); @@ -205,33 +190,24 @@ public void testMultiBucketAggregationAsSubAggregation() throws IOException { Consumer verifier = ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat( - ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg}").docCount, - equalTo(2L) - ); - InternalDateHistogram byTimeStampBucket = ts.getBucketByKey( - "{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg}" - ).getAggregations().get("by_timestamp"); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); + InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg") + .getAggregations() + .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); - assertThat( - ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4}").docCount, - equalTo(2L) - ); - byTimeStampBucket = ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4}") + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); + byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4") .getAggregations() .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); - assertThat( - ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o}").docCount, - equalTo(4L) - ); - byTimeStampBucket = ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o}") + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); + byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o") .getAggregations() .get("by_timestamp"); assertThat( @@ -253,24 +229,9 @@ public void testAggregationSize() throws IOException { List> verifiers = new ArrayList>(); - verifiers.add( - ts -> assertThat( - ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAAAS_0VrZGalylFPi9dkK4dYyY9g0yybS6o}").docCount, - equalTo(4L) - ) - ); - verifiers.add( - ts -> assertThat( - ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg}").docCount, - equalTo(2L) - ) - ); - verifiers.add( - ts -> assertThat( - ts.getBucketByKey("{_tsid=C0uMhZuLQSpAQ5ipTZFLMgAAAAAAAAAAAAAAAAAAAABjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4}").docCount, - equalTo(2L) - ) - ); + verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L))); for (int i = 1; i <= 3; i++) { int size = i; diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index e1393c4213fe8..275bd446d3996 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -2,8 +2,8 @@ missing routing path field: - skip: features: close_to - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -71,23 +71,19 @@ missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key._tsid: "3LQFBfjW0o1rcoR2NF5cz-ix0Ruaua4Wo_hP7e-sEqo5ihHDIG1DaFBdVI_fYOQvJgKOvg" } - - match: { aggregations.tsids.buckets.0.key.tag: null } + - match: { aggregations.tsids.buckets.0.key: "JNy0BQX41tKNa3KEdjReXM85ihHDIG1DaFBdVI_fYOQvJgKOvg" } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "3LQFBfjW0o1rcoR2NF5cz-ix0Ruaua4Wo_hP7e-sEqp12oDh9NI69d0Kk5TQ6CAdewYP5A" } - - match: { aggregations.tsids.buckets.1.key.tag: null } + - match: { aggregations.tsids.buckets.1.key: "JNy0BQX41tKNa3KEdjReXM912oDh9NI69d0Kk5TQ6CAdewYP5A" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.15, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "M4NGZuPu-7ista-OslulS-ix0Ruaua4Wo_hP7e-sEqqaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } - - match: { aggregations.tsids.buckets.2.key.tag: null } + - match: { aggregations.tsids.buckets.2.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "M4NGZuPu-7ista-OslulS-ix0Ruaua4Wo_hP7e-sEqrcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } - - match: { aggregations.tsids.buckets.3.key.tag: null } + - match: { aggregations.tsids.buckets.3.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 7.30, error: 0.01 }} @@ -202,12 +198,10 @@ multi-value routing path field: - match: {hits.total.value: 4} - length: {aggregations.tsids.buckets: 2} - - match: {aggregations.tsids.buckets.0.key._tsid: "M4NGZuPu-7ista-OslulS-ix0Ruaua4Wo_hP7e-sEqqaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } - - match: {aggregations.tsids.buckets.0.key.tag: null } + - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } - match: {aggregations.tsids.buckets.0.doc_count: 2 } - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "M4NGZuPu-7ista-OslulS-ix0Ruaua4Wo_hP7e-sEqrcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } - - match: { aggregations.tsids.buckets.1.key.tag: null } + - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } - match: {aggregations.tsids.buckets.1.doc_count: 2 } - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index adedd4588b53a..4f9089533ba8e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -1,9 +1,3 @@ ---- -setup: - - skip: - version: "8.7.00 - 8.11.99" - reason: _tsid hasing introduced in 8.12 - --- add dimensions with put_mapping: - skip: @@ -54,7 +48,7 @@ add dimensions with put_mapping: - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ { _tsid: "27hcKTYkXCOfYiuRWRTWygAAAAAAAAAAAAAAAAAAAAAGlT_5e6_NYGOZWULpmMG9IAlZlA" } ] } + - match: {hits.hits.0.fields._tsid: [ "JNu4XCk2JFwjn2IrkVkU1soGlT_5e6_NYGOZWULpmMG9IAlZlA" ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 478237b6a19f1..74eb8ea9ad4fa 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -1,9 +1,3 @@ ---- -setup: - - skip: - version: "8.7.00 - 8.9.99" - reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - --- keyword dimension: - skip: @@ -71,10 +65,10 @@ keyword dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "3LQFBfjW0o1rcoR2NF5czwAAAAAAAAAAAAAAAAAAAAA5ihHDIG1DaFBdVI_fYOQvJgKOvg"}} + - match: {aggregations.tsids.buckets.0.key: "JNy0BQX41tKNa3KEdjReXM85ihHDIG1DaFBdVI_fYOQvJgKOvg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "3LQFBfjW0o1rcoR2NF5czwAAAAAAAAAAAAAAAAAAAAB12oDh9NI69d0Kk5TQ6CAdewYP5A"}} + - match: {aggregations.tsids.buckets.1.key: "JNy0BQX41tKNa3KEdjReXM912oDh9NI69d0Kk5TQ6CAdewYP5A"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -150,35 +144,19 @@ flattened dimension: - match: { hits.total.value: 8} - length: { aggregations.tsids.buckets: 4} - - match: { aggregations.tsids.buckets.0.key._tsid: "ItgQI_4ahoh-OTREF92WDQAAAAAAAAAAAAAAAAAAAAAoaZ29eRDHR3kQx0fjCuTKddqA4R9ytcYdqdbcoJ8VBuvqFtQ" } - - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: null } - - match: { aggregations.tsids.buckets.0.key.deployment\.version\.minor: null } - - match: { aggregations.tsids.buckets.0.key.deployment\.version\.patch: null } + - match: { aggregations.tsids.buckets.0.key: "NCLYECP-GoaIfjk0RBfdlg0oaZ29eRDHR3kQx0fjCuTKddqA4R9ytcYdqdbcoJ8VBuvqFtQ" } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "ItgQI_4ahoh-OTREF92WDQAAAAAAAAAAAAAAAAAAAACrnf7qeRDHR3kQx0eLDvTuOYoRw4EeUIKK1KGFPmxqCWQyQJE" } - - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: null } - - match: { aggregations.tsids.buckets.1.key.deployment\.version\.minor: null } - - match: { aggregations.tsids.buckets.1.key.deployment\.version\.patch: null } + - match: { aggregations.tsids.buckets.1.key: "NCLYECP-GoaIfjk0RBfdlg2rnf7qeRDHR3kQx0eLDvTuOYoRw4EeUIKK1KGFPmxqCWQyQJE" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "ItgQI_4ahoh-OTREF92WDQAAAAAAAAAAAAAAAAAAAADP1l1UeRDHR3kQx0fjCuTKddqA4ai7CiSuCxA-PhBdLYOXkVY" } - - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: null } - - match: { aggregations.tsids.buckets.2.key.deployment\.version\.minor: null } - - match: { aggregations.tsids.buckets.2.key.deployment\.version\.patch: null } + - match: { aggregations.tsids.buckets.2.key: "NCLYECP-GoaIfjk0RBfdlg3P1l1UeRDHR3kQx0fjCuTKddqA4ai7CiSuCxA-PhBdLYOXkVY" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "ItgQI_4ahoh-OTREF92WDQAAAAAAAAAAAAAAAAAAAADgKelzeRDHR3kQx0eLDvTuOYoRw6Wapg_P07YIhdV3NFJDjjE" } - - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: null } - - match: { aggregations.tsids.buckets.3.key.deployment\.version\.minor: null } - - match: { aggregations.tsids.buckets.3.key.deployment\.version\.patch: null } + - match: { aggregations.tsids.buckets.3.key: "NCLYECP-GoaIfjk0RBfdlg3gKelzeRDHR3kQx0eLDvTuOYoRw6Wapg_P07YIhdV3NFJDjjE" } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.65, error: 0.01 }} @@ -254,11 +232,11 @@ flattened empty dimension: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 2 } - - match: { aggregations.tsids.buckets.0.key._tsid: "3LQFBfjW0o1rcoR2NF5czwAAAAAAAAAAAAAAAAAAAAA5ihHDIG1DaFBdVI_fYOQvJgKOvg" } + - match: { aggregations.tsids.buckets.0.key: "JNy0BQX41tKNa3KEdjReXM85ihHDIG1DaFBdVI_fYOQvJgKOvg" } - match: { aggregations.tsids.buckets.0.doc_count: 4 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "3LQFBfjW0o1rcoR2NF5czwAAAAAAAAAAAAAAAAAAAAB12oDh9NI69d0Kk5TQ6CAdewYP5A" } + - match: { aggregations.tsids.buckets.1.key: "JNy0BQX41tKNa3KEdjReXM912oDh9NI69d0Kk5TQ6CAdewYP5A" } - match: { aggregations.tsids.buckets.1.doc_count: 4 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.22, error: 0.01 }} @@ -334,45 +312,27 @@ flattened field missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 6 } - - match: { aggregations.tsids.buckets.0.key._tsid: "8HNIqk_KxgNH6XWO9gDyMwAAAAAAAAAAAAAAAAAAAADxTXebeRDHRzmKEcOUW9u_kzgC7pSzoi1utVcm" } - - match: { aggregations.tsids.buckets.0.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.0.key.deployment\.build\.branch: null } - - match: { aggregations.tsids.buckets.0.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.0.key.: "LPBzSKpPysYDR-l1jvYA8jPxTXebeRDHRzmKEcOUW9u_kzgC7pSzoi1utVcm" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "8HNIqk_KxgNH6XWO9gDyMwAAAAAAAAAAAAAAAAAAAADxTXebeRDHR3XagOEjGQJfwIB2Q5kLDL76leH4" } - - match: { aggregations.tsids.buckets.1.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.1.key.deployment\.build\.branch: null } - - match: { aggregations.tsids.buckets.1.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.1.key: "LPBzSKpPysYDR-l1jvYA8jPxTXebeRDHR3XagOEjGQJfwIB2Q5kLDL76leH4" } - match: { aggregations.tsids.buckets.1.doc_count: 1 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "XXEWRB81VefgRtsBbneB7wAAAAAAAAAAAAAAAAAAAADxTXebKGmdvXkQx0d12oDh16FtfvP_ObklCa7gXDcmdA" } - - match: { aggregations.tsids.buckets.2.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.2.key.deployment\.build\.branch: null } - - match: { aggregations.tsids.buckets.2.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.2.key: "MF1xFkQfNVXn4EbbAW53ge_xTXebKGmdvXkQx0d12oDh16FtfvP_ObklCa7gXDcmdA" } - match: { aggregations.tsids.buckets.2.doc_count: 1 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "XXEWRB81VefgRtsBbneB7wAAAAAAAAAAAAAAAAAAAADxTXebq53-6nkQx0c5ihHDU3PuzTI1mM98gQqqihDoZA" } - - match: { aggregations.tsids.buckets.3.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.3.key.deployment\.build\.branch: null } - - match: { aggregations.tsids.buckets.3.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.3.key: "MF1xFkQfNVXn4EbbAW53ge_xTXebq53-6nkQx0c5ihHDU3PuzTI1mM98gQqqihDoZA" } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.4.key._tsid: "XXEWRB81VefgRtsBbneB7wAAAAAAAAAAAAAAAAAAAADxTXebz9ZdVHkQx0d12oDhi1SccpVnRAhpjCrWhqFjfQ" } - - match: { aggregations.tsids.buckets.4.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.4.key.deployment\.build\.branch: null } - - match: { aggregations.tsids.buckets.4.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.4.key: "MF1xFkQfNVXn4EbbAW53ge_xTXebz9ZdVHkQx0d12oDhi1SccpVnRAhpjCrWhqFjfQ" } - match: { aggregations.tsids.buckets.4.doc_count: 2 } - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.5.key._tsid: "XXEWRB81VefgRtsBbneB7wAAAAAAAAAAAAAAAAAAAADxTXeb4Cnpc3kQx0c5ihHDqaS5hHgLAm0E3XkHG7fdkQ" } - - match: { aggregations.tsids.buckets.5.key.deployment\.build\.tag: null } - - match: { aggregations.tsids.buckets.5.key.deployment\.build\.branch: null } - - match: { aggregations.tsids.buckets.5.key.deployment\.version\.major: null } + - match: { aggregations.tsids.buckets.5.key: "MF1xFkQfNVXn4EbbAW53ge_xTXeb4Cnpc3kQx0c5ihHDqaS5hHgLAm0E3XkHG7fdkQ" } - match: { aggregations.tsids.buckets.5.doc_count: 1 } - close_to: { aggregations.tsids.buckets.5.voltage.value: { value: 6.60, error: 0.01 }} @@ -452,19 +412,19 @@ flattened field misspelled routing path field: - match: { hits.total.value: 6 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key._tsid: "GzT4lnbKgCD4uvDfky7fKwAAAAAAAAAAAAAAAAAAAAAoaZ29egRdyNeHXPSPghDVzguaRg" } + - match: { aggregations.tsids.buckets.0.key: "JBs0-JZ2yoAg-Lrw35Mu3ysoaZ29egRdyNeHXPSPghDVzguaRg" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key._tsid: "GzT4lnbKgCD4uvDfky7fKwAAAAAAAAAAAAAAAAAAAACrnf7qs9-VXFZ6jjZCbl_iiXSs7Q" } + - match: { aggregations.tsids.buckets.1.key: "JBs0-JZ2yoAg-Lrw35Mu3yurnf7qs9-VXFZ6jjZCbl_iiXSs7Q" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key._tsid: "GzT4lnbKgCD4uvDfky7fKwAAAAAAAAAAAAAAAAAAAADP1l1UlmlXEQVNXrHpUvpn7by0jA" } + - match: { aggregations.tsids.buckets.2.key: "JBs0-JZ2yoAg-Lrw35Mu3yvP1l1UlmlXEQVNXrHpUvpn7by0jA" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key._tsid: "GzT4lnbKgCD4uvDfky7fKwAAAAAAAAAAAAAAAAAAAADgKelz9WSJqzeYh7aza_7yxDXMZA" } + - match: { aggregations.tsids.buckets.3.key: "JBs0-JZ2yoAg-Lrw35Mu3yvgKelz9WSJqzeYh7aza_7yxDXMZA" } - match: { aggregations.tsids.buckets.3.doc_count: 1 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} @@ -536,10 +496,10 @@ long dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "xq55J0GFz9YiFjjGg8TYRwAAAAAAAAAAAAAAAAAAAABDgKYd73outpXn7LJV-gQfvlrec7NyMho"}} + - match: {aggregations.tsids.buckets.0.key: "KMaueSdBhc_WIhY4xoPE2EdDgKYd73outpXn7LJV-gQfvlrec7NyMho"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "xq55J0GFz9YiFjjGg8TYRwAAAAAAAAAAAAAAAAAAAACtm41v73outmoSTcFmfQBYjOjMaOWM5zs"}} + - match: {aggregations.tsids.buckets.1.key: "KMaueSdBhc_WIhY4xoPE2Eetm41v73outmoSTcFmfQBYjOjMaOWM5zs"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -611,10 +571,10 @@ ip dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: { _tsid: "tWt3sTyhkvJ3619KbQfqMAAAAAAAAAAAAAAAAAAAAADYzb-v73outosB-eDb_nzTrIVsJFVQR2c"}} + - match: {aggregations.tsids.buckets.0.key: "KLVrd7E8oZLyd-tfSm0H6jDYzb-v73outosB-eDb_nzTrIVsJFVQR2c"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: { _tsid: "tWt3sTyhkvJ3619KbQfqMAAAAAAAAAAAAAAAAAAAAADaaOFK73outkqpG8R65Gm4VUhpyuc11zw"}} + - match: {aggregations.tsids.buckets.1.key: "KLVrd7E8oZLyd-tfSm0H6jDaaOFK73outkqpG8R65Gm4VUhpyuc11zw"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index 0a83867aebf28..4e3bc2510e55c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -102,8 +102,8 @@ split: --- shrink: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.shrink: @@ -124,13 +124,13 @@ shrink: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}]} + - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} --- clone: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.clone: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml index ef34e64ad41d7..2c99a691f127b 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml @@ -63,8 +63,8 @@ setup: --- aggretate multi_terms: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: size: 0 @@ -78,11 +78,11 @@ aggretate multi_terms: - field: k8s.pod.ip - length: { aggregations.m_terms.buckets: 3 } - - match: { aggregations.m_terms.buckets.0.key_as_string: "{k8s.pod.uid=df3145b3-0563-4d3b-a0f7-897eb2876ea9, metricset=pod}|10.10.55.3" } + - match: { aggregations.m_terms.buckets.0.key_as_string: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ|10.10.55.3" } - match: { aggregations.m_terms.buckets.0.doc_count: 4 } - - match: { aggregations.m_terms.buckets.1.key_as_string: "{k8s.pod.uid=947e4ced-1786-4e53-9e0c-5c447e959507, metricset=pod}|10.10.55.1" } + - match: { aggregations.m_terms.buckets.1.key_as_string: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o|10.10.55.1" } - match: { aggregations.m_terms.buckets.1.doc_count: 3 } - - match: { aggregations.m_terms.buckets.2.key_as_string: "{k8s.pod.uid=947e4ced-1786-4e53-9e0c-5c447e959507, metricset=pod}|10.10.55.2" } + - match: { aggregations.m_terms.buckets.2.key_as_string: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o|10.10.55.2" } - match: { aggregations.m_terms.buckets.2.doc_count: 1 } --- From 677bb21fb59198f6f99a9fb7fd08528fd55eea24 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Nov 2023 10:32:52 +0100 Subject: [PATCH 035/125] fix: some more tsid tests --- .../metrics/geoline-aggregation.asciidoc | 18 ++++++------------ .../rest-api-spec/test/delete/70_tsdb.yml | 8 ++++---- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc b/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc index aabe8d172e4a0..98b3593ab1ba9 100644 --- a/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc @@ -347,10 +347,8 @@ This query will result in: "aggregations": { "path": { "buckets": { - "{city=Paris}": { - "key": { - "city": "Paris" - }, + "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": { + "key": "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA", "doc_count": 2, "museum_tour": { "type": "Feature", @@ -363,10 +361,8 @@ This query will result in: } } }, - "{city=Antwerp}": { - "key": { - "city": "Antwerp" - }, + "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": { + "key": "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw", "doc_count": 3, "museum_tour": { "type": "Feature", @@ -379,10 +375,8 @@ This query will result in: } } }, - "{city=Amsterdam}": { - "key": { - "city": "Amsterdam" - }, + "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": { + "key": "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA", "doc_count": 5, "museum_tour": { "type": "Feature", diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml index 82e064304ce4c..ce61e0a0711d0 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml @@ -43,15 +43,15 @@ location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhC8WO3aVdhvtAAABiHD35_g } + - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } - match: { result: created } - match: { _version: 1 } - do: delete: index: weather_sensors - id: crxuhDzuYSDy965GAAABiHD35_g - - match: { _id: crxuhDzuYSDy965GAAABiHD35_g } + id: crxuhAep5Npwt_etAAABiHD35_g + - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } - match: { result: deleted } - match: { _version: 2 } @@ -68,6 +68,6 @@ location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhDzuYSDy965GAAABiHD35_g } + - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } - match: { result: created } - match: { _version: 3 } From 4d5fd77357f714e0f19ca960385385edf27a8f54 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Nov 2023 12:11:33 +0100 Subject: [PATCH 036/125] fix: some more tests and ParsedTimeSeries --- .../bucket/timeseries/ParsedTimeSeries.java | 5 ++--- .../timeseries/InternalTimeSeriesTests.java | 4 ++-- .../rest-api-spec/test/tsdb/100_composite.yml | 16 ++++++++-------- .../rest-api-spec/test/tsdb/50_alias.yml | 14 +++++++------- .../org/elasticsearch/search/DocValueFormat.java | 1 + .../composite/CompositeAggregationBuilder.java | 5 ++++- 6 files changed, 24 insertions(+), 21 deletions(-) diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/ParsedTimeSeries.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/ParsedTimeSeries.java index f4eff3e878b59..9a45c4554137c 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/ParsedTimeSeries.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/ParsedTimeSeries.java @@ -16,7 +16,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; public class ParsedTimeSeries extends ParsedMultiBucketAggregation { @@ -63,7 +62,7 @@ public static ParsedTimeSeries fromXContent(XContentParser parser, String name) static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket { - private Map key; + private Object key; @Override public Object getKey() { @@ -76,7 +75,7 @@ public String getKeyAsString() { } static ParsedTimeSeries.ParsedBucket fromXContent(XContentParser parser, boolean keyed) throws IOException { - return parseXContent(parser, keyed, ParsedTimeSeries.ParsedBucket::new, (p, bucket) -> bucket.key = new TreeMap<>(p.map())); + return parseXContent(parser, keyed, ParsedTimeSeries.ParsedBucket::new, (p, bucket) -> bucket.key = p.text()); } } diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java index 1d9a48a1fd923..e1674e881f8eb 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java @@ -91,10 +91,10 @@ protected InternalTimeSeries createTestInstance(String name, Map @Override @SuppressWarnings("unchecked") protected void assertReduced(InternalTimeSeries reduced, List inputs) { - Map, Long> keys = new HashMap<>(); + Map keys = new HashMap<>(); for (InternalTimeSeries in : inputs) { for (InternalBucket bucket : in.getBuckets()) { - keys.compute((Map) bucket.getKey(), (k, v) -> { + keys.compute((String) bucket.getKey(), (k, v) -> { if (v == null) { return bucket.docCount; } else { diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml index 19d3defba1a4a..8362dee534656 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml @@ -64,8 +64,8 @@ setup: --- composite aggregation on tsid: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: @@ -139,17 +139,17 @@ composite aggregation on tsid with after: } ] after: { - tsid: { k8s.pod.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9", metricset: "pod" }, - date: 1619635800000 + tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o", + date: 1619635860000 } - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 1 } - - match: { aggregations.tsids.buckets.0.key.tsid.k8s\.pod\.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.0.key.tsid.metricset: "pod" } + + - match: { aggregations.tsids.buckets.0.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } - match: { aggregations.tsids.buckets.0.key.date: 1619635860000} - match: { aggregations.tsids.buckets.0.doc_count: 1 } - - match: { aggregations.tsids.after_key.tsid.k8s\.pod\.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.after_key.tsid.metricset: "pod" } + + - match: { aggregations.tsids.after_key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.after_key.date: 1619635860000} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index a36bd16647b93..c1b304cb1c155 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -85,16 +85,16 @@ search an alias: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "AgtrOHMucG9kLnVpZHMkOTQ3ZTRjZWQtMTc4Ni00ZTUzLTllMGMtNWM0NDdlOTU5NTA3CW1ldHJpY3NldHMDcG9k"} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "AgtrOHMucG9kLnVpZHMkOTQ3ZTRjZWQtMTc4Ni00ZTUzLTllMGMtNWM0NDdlOTU5NTA3CW1ldHJpY3NldHMDcG9k"} + - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- index into alias: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.put_alias: @@ -129,10 +129,10 @@ index into alias: _key: asc - match: {hits.total.value: 12} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} + - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - match: {aggregations.tsids.buckets.2.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAC-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"}} + - match: {aggregations.tsids.buckets.2.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.2.doc_count: 4} diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 2fa28d555fc60..18179a9603c0e 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -739,6 +739,7 @@ private BytesRef parseBytesRefMap(Object value) { } try { + // NOTE: we can decode the tsid only if it is not hashed (represented as a map) return builder.withoutHash().toBytesRef(); } catch (IOException e) { throw new IllegalArgumentException(e); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java index ca88d50898763..98f27b0a0ce49 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java @@ -246,11 +246,14 @@ protected AggregatorFactory doBuild( Object obj = after.get(sourceName); if (configs[i].missingBucket() && obj == null) { values[i] = null; - } else if (obj instanceof Comparable c) { + } else if (obj instanceof Comparable c && configs[i].fieldType().getClass() != TimeSeriesIdFieldType.class) { values[i] = c; } else if (obj instanceof Map && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { // If input is a _tsid map, encode the map to the _tsid BytesRef values[i] = configs[i].format().parseBytesRef(obj); + } else if (obj instanceof String && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { + // If input is a _tsid string, encode the string to the _tsid BytesRef + values[i] = configs[i].format().parseBytesRef(obj); } else { throw new IllegalArgumentException( "Invalid value for [after." From 64c717240152e5b48dedc080a934413ad0be4787 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Nov 2023 13:00:58 +0100 Subject: [PATCH 037/125] fix: geo line tests and a few more tsid values in composite --- .../rest-api-spec/test/tsdb/100_composite.yml | 26 +++++++++---------- .../aggregations/GeoLineAggregatorTests.java | 12 ++++++--- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml index 8362dee534656..88874cc6126d9 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml @@ -91,31 +91,31 @@ composite aggregation on tsid: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key.tsid.k8s\.pod\.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.0.key.tsid.metricset: "pod" } + + - match: { aggregations.tsids.buckets.0.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } - match: { aggregations.tsids.buckets.0.key.date: 1619635800000} - match: { aggregations.tsids.buckets.0.doc_count: 3 } - - match: { aggregations.tsids.buckets.1.key.tsid.k8s\.pod\.uid: "947e4ced-1786-4e53-9e0c-5c447e959507" } - - match: { aggregations.tsids.buckets.1.key.tsid.metricset: "pod" } + + - match: { aggregations.tsids.buckets.1.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } - match: { aggregations.tsids.buckets.1.key.date: 1619635860000} - match: { aggregations.tsids.buckets.1.doc_count: 1 } - - match: { aggregations.tsids.buckets.2.key.tsid.k8s\.pod\.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.2.key.tsid.metricset: "pod" } - - match: { aggregations.tsids.buckets.2.key.date: 1619635800000} + + - match: { aggregations.tsids.buckets.2.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } + - match: { aggregations.tsids.buckets.2.key.date: 1619635800000 } - match: { aggregations.tsids.buckets.2.doc_count: 3 } - - match: { aggregations.tsids.buckets.3.key.tsid.k8s\.pod\.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.buckets.3.key.tsid.metricset: "pod" } + + - match: { aggregations.tsids.buckets.3.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.buckets.3.key.date: 1619635860000} - match: { aggregations.tsids.buckets.3.doc_count: 1 } - - match: { aggregations.tsids.after_key.tsid.k8s\.pod\.uid: "df3145b3-0563-4d3b-a0f7-897eb2876ea9" } - - match: { aggregations.tsids.after_key.tsid.metricset: "pod" } + + - match: { aggregations.tsids.after_key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.after_key.date: 1619635860000} --- composite aggregation on tsid with after: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index b9e073c3d70dd..b61dbe1e4ae3c 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -355,7 +355,8 @@ public void testGeoLine_TSDB() throws IOException { assertThat("Number of time-series buckets", tsx.ts.getBuckets().size(), equalTo(g)); for (int i = 0; i < tsx.groups.length; i++) { InternalGeoLine geoLine = tsx.ts.getBuckets().get(i).getAggregations().get("track"); - assertGeoLine(sortOrder, tsx.groups[i], geoLine, tsx, true); + int index = testConfig.useTimeSeriesAggregation ? tsx.groups.length - i - 1 : i; + assertGeoLine(sortOrder, tsx.groups[index], geoLine, tsx, true); } }); } @@ -378,7 +379,8 @@ public void testGeoLine_Terms_TSDB() throws IOException { StringTerms terms = tsx.ts.getBuckets().get(i).getAggregations().get("groups"); assertThat("Number of terms buckets", terms.getBuckets().size(), equalTo(1)); InternalGeoLine geoLine = terms.getBuckets().get(0).getAggregations().get("track"); - assertGeoLine(sortOrder, tsx.groups[i], geoLine, tsx, true); + int index = testConfig.useTimeSeriesAggregation ? tsx.groups.length - i - 1 : 0; + assertGeoLine(sortOrder, tsx.groups[index], geoLine, tsx, true); } }); } @@ -398,7 +400,8 @@ public void testGeoLine_TSDB_simplified() throws IOException { assertThat("Number of time-series buckets", tsx.ts.getBuckets().size(), equalTo(g)); for (int i = 0; i < tsx.groups.length; i++) { InternalGeoLine geoLine = tsx.ts.getBuckets().get(i).getAggregations().get("track"); - assertGeoLine(sortOrder, tsx.groups[i], geoLine, tsx, false); + int index = testConfig.useTimeSeriesAggregation ? tsx.groups.length - i - 1 : i; + assertGeoLine(sortOrder, tsx.groups[index], geoLine, tsx, false); } }); } @@ -421,7 +424,8 @@ public void testGeoLine_Terms_TSDB_simplified() throws IOException { StringTerms terms = tsx.ts.getBuckets().get(i).getAggregations().get("groups"); assertThat("Number of terms buckets", terms.getBuckets().size(), equalTo(1)); InternalGeoLine geoLine = terms.getBuckets().get(0).getAggregations().get("track"); - assertGeoLine(sortOrder, tsx.groups[i], geoLine, tsx, false); + int index = testConfig.useTimeSeriesAggregation ? tsx.groups.length - i - 1 : 0; + assertGeoLine(sortOrder, tsx.groups[index], geoLine, tsx, false); } }); } From ca9f6680cdbc7a40c47586c65d360419678d0b87 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Nov 2023 14:50:33 +0100 Subject: [PATCH 038/125] fix: downsampling tsid values and bucket ordering --- .../test/data_stream/150_tsdb.yml | 14 +- .../rest-api-spec/test/tsdb/40_search.yml | 4 +- .../test/downsample/10_basic.yml | 309 +++++++++--------- 3 files changed, 164 insertions(+), 163 deletions(-) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index a0e67c61d076e..442ba800200a2 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -133,8 +133,8 @@ created the data stream: --- fetch the tsid: - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: @@ -147,13 +147,13 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [{_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" }]} + - match: {hits.hits.0.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" ]} --- "aggregate the tsid": - skip: - version: " - 8.0.99" - reason: introduced in 8.1.0 + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: search: @@ -168,9 +168,9 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} + - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index b280e2f901a17..3e094cd2eb1ae 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -282,9 +282,9 @@ aggregate a tag: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAA5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: {_tsid: "KMQn1H8GA7xNFfZA53p2lAAAAAAAAAAAAAAAAAAAAAB12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"}} + - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index 70f66f38d39b9..7a6e364ebba6a 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -289,8 +289,8 @@ setup: --- "Downsample index": - skip: - version: " - 8.4.99" - reason: "Downsampling GA-ed in 8.7.0" + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.downsample: @@ -310,23 +310,23 @@ setup: - length: { hits.hits: 4 } - match: { hits.hits.0._source._doc_count: 2 } - - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - match: { hits.hits.0._source.metricset: pod } - - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } - - match: { hits.hits.0._source.k8s.pod.multi-counter: 21 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 90 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 200 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 726 } + - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.0._source.k8s.pod.multi-counter: 0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 100.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 102.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 607.0 } - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 } - - match: { hits.hits.0._source.k8s.pod.network.tx.min: 2001818691 } - - match: { hits.hits.0._source.k8s.pod.network.tx.max: 2005177954 } + - match: { hits.hits.0._source.k8s.pod.network.tx.min: 1434521831 } + - match: { hits.hits.0._source.k8s.pod.network.tx.max: 1434577921 } - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 } - - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.26" } - - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:35:00.000Z" } - - match: { hits.hits.0._source.k8s.pod.number_of_containers: 2 } - - match: { hits.hits.0._source.k8s.pod.tags: ["backend", "prod", "us-west1"] } - - match: { hits.hits.0._source.k8s.pod.values: [1, 1, 3] } - - is_true: hits.hits.0._source.k8s.pod.running + - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.56" } + - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:43:00.000Z" } + - match: { hits.hits.0._source.k8s.pod.number_of_containers: 1 } + - match: { hits.hits.0._source.k8s.pod.tags: ["backend", "test", "us-west2"] } + - match: { hits.hits.0._source.k8s.pod.values: [1, 1, 2] } + - is_false: hits.hits.0._source.k8s.pod.running # Assert rollup index settings - do: @@ -678,8 +678,8 @@ setup: --- "Downsample a downsampled index": - skip: - version: " - 8.6.99" - reason: "Rollup GA-ed in 8.7.0" + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.downsample: @@ -730,33 +730,33 @@ setup: sort: [ "_tsid", "@timestamp" ] - length: { hits.hits: 3 } - - match: { hits.hits.0._source._doc_count: 2 } - - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + - match: { hits.hits.0._source._doc_count: 4 } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - match: { hits.hits.0._source.metricset: pod } - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } - - match: { hits.hits.0._source.k8s.pod.multi-counter: 21 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 90 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 200 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 726 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 } - - match: { hits.hits.0._source.k8s.pod.network.tx.min: 2001818691 } - - match: { hits.hits.0._source.k8s.pod.network.tx.max: 2005177954 } - - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 } - - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.26" } - - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:35:00.000Z" } - - match: { hits.hits.0._source.k8s.pod.number_of_containers: 2 } - - match: { hits.hits.0._source.k8s.pod.tags: [ "backend", "prod", "us-west1" ] } - - match: { hits.hits.0._source.k8s.pod.values: [ 1, 1, 3 ] } + - match: { hits.hits.0._source.k8s.pod.multi-counter: 76 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 95.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 110.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 1209.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 12 } + - match: { hits.hits.0._source.k8s.pod.network.tx.min: 1434521831 } + - match: { hits.hits.0._source.k8s.pod.network.tx.max: 1434595272 } + - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 4 } + - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.120" } + - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:45:00.000Z" } + - match: { hits.hits.0._source.k8s.pod.number_of_containers: 1 } + - match: { hits.hits.0._source.k8s.pod.tags: [ "backend", "test", "us-west1" ] } + - match: { hits.hits.0._source.k8s.pod.values: [ 1, 2, 3 ] } - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - match: { hits.hits.1._source.metricset: pod } - - match: { hits.hits.1._source.@timestamp: 2021-04-28T20:00:00.000Z } + - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:00:00.000Z } - match: { hits.hits.1._source._doc_count: 2 } - - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - match: { hits.hits.2._source.metricset: pod } - - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z } - - match: { hits.hits.2._source._doc_count: 4 } + - match: { hits.hits.2._source.@timestamp: 2021-04-28T20:00:00.000Z } + - match: { hits.hits.2._source._doc_count: 2 } - do: indices.downsample: @@ -817,8 +817,8 @@ setup: --- "Downsample histogram as label": - skip: - version: " - 8.4.99" - reason: "rollup renamed to downsample in 8.5.0" + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.downsample: @@ -855,65 +855,66 @@ setup: sort: [ "_tsid", "@timestamp" ] - length: { hits.hits: 4 } - - match: { hits.hits.0._source._doc_count: 2 } - - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.0._source.metricset: pod } - - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } + + - match: { hits.hits.0._source._doc_count: 2 } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.0._source.metricset: pod } + - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } - length: { hits.hits.0._source.k8s.pod.latency.counts: 4 } - - match: { hits.hits.0._source.k8s.pod.latency.counts.0: 8 } - - match: { hits.hits.0._source.k8s.pod.latency.counts.1: 7 } - - match: { hits.hits.0._source.k8s.pod.latency.counts.2: 10 } - - match: { hits.hits.0._source.k8s.pod.latency.counts.3: 12 } + - match: { hits.hits.0._source.k8s.pod.latency.counts.0: 2 } + - match: { hits.hits.0._source.k8s.pod.latency.counts.1: 2 } + - match: { hits.hits.0._source.k8s.pod.latency.counts.2: 8 } + - match: { hits.hits.0._source.k8s.pod.latency.counts.3: 8 } - length: { hits.hits.0._source.k8s.pod.latency.values: 4 } - - match: { hits.hits.0._source.k8s.pod.latency.values.0: 1.0 } - - match: { hits.hits.0._source.k8s.pod.latency.values.1: 2.0 } - - match: { hits.hits.0._source.k8s.pod.latency.values.2: 5.0 } - - match: { hits.hits.0._source.k8s.pod.latency.values.3: 10.0 } - - - match: { hits.hits.1._source._doc_count: 2 } - - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.1._source.metricset: pod } - - match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z } + - match: { hits.hits.0._source.k8s.pod.latency.values.0: 1.0 } + - match: { hits.hits.0._source.k8s.pod.latency.values.1: 10.0 } + - match: { hits.hits.0._source.k8s.pod.latency.values.2: 100.0 } + - match: { hits.hits.0._source.k8s.pod.latency.values.3: 1000.0 } + + - match: { hits.hits.1._source._doc_count: 1 } + - match: { hits.hits.1._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.1._source.metricset: pod } + - match: { hits.hits.1._source.@timestamp: 2021-04-28T19:00:00.000Z } - length: { hits.hits.1._source.k8s.pod.latency.counts: 4 } - - match: { hits.hits.1._source.k8s.pod.latency.counts.0: 7 } - - match: { hits.hits.1._source.k8s.pod.latency.counts.1: 15 } - - match: { hits.hits.1._source.k8s.pod.latency.counts.2: 10 } - - match: { hits.hits.1._source.k8s.pod.latency.counts.3: 10 } + - match: { hits.hits.1._source.k8s.pod.latency.counts.0: 4 } + - match: { hits.hits.1._source.k8s.pod.latency.counts.1: 5 } + - match: { hits.hits.1._source.k8s.pod.latency.counts.2: 4 } + - match: { hits.hits.1._source.k8s.pod.latency.counts.3: 13 } - length: { hits.hits.1._source.k8s.pod.latency.values: 4 } - - match: { hits.hits.1._source.k8s.pod.latency.values.0: 1.0 } - - match: { hits.hits.1._source.k8s.pod.latency.values.1: 2.0 } - - match: { hits.hits.1._source.k8s.pod.latency.values.2: 5.0 } - - match: { hits.hits.1._source.k8s.pod.latency.values.3: 10.0 } + - match: { hits.hits.1._source.k8s.pod.latency.values.0: 1.0 } + - match: { hits.hits.1._source.k8s.pod.latency.values.1: 10.0 } + - match: { hits.hits.1._source.k8s.pod.latency.values.2: 100.0 } + - match: { hits.hits.1._source.k8s.pod.latency.values.3: 1000.0 } - match: { hits.hits.2._source._doc_count: 2 } - - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - match: { hits.hits.2._source.metricset: pod } - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z } - length: { hits.hits.2._source.k8s.pod.latency.counts: 4 } - - match: { hits.hits.2._source.k8s.pod.latency.counts.0: 2 } - - match: { hits.hits.2._source.k8s.pod.latency.counts.1: 2 } - - match: { hits.hits.2._source.k8s.pod.latency.counts.2: 8 } - - match: { hits.hits.2._source.k8s.pod.latency.counts.3: 8 } + - match: { hits.hits.2._source.k8s.pod.latency.counts.0: 8 } + - match: { hits.hits.2._source.k8s.pod.latency.counts.1: 7 } + - match: { hits.hits.2._source.k8s.pod.latency.counts.2: 10 } + - match: { hits.hits.2._source.k8s.pod.latency.counts.3: 12 } - length: { hits.hits.2._source.k8s.pod.latency.values: 4 } - match: { hits.hits.2._source.k8s.pod.latency.values.0: 1.0 } - - match: { hits.hits.2._source.k8s.pod.latency.values.1: 10.0 } - - match: { hits.hits.2._source.k8s.pod.latency.values.2: 100.0 } - - match: { hits.hits.2._source.k8s.pod.latency.values.3: 1000.0 } + - match: { hits.hits.2._source.k8s.pod.latency.values.1: 2.0 } + - match: { hits.hits.2._source.k8s.pod.latency.values.2: 5.0 } + - match: { hits.hits.2._source.k8s.pod.latency.values.3: 10.0 } - - match: { hits.hits.3._source._doc_count: 1 } - - match: { hits.hits.3._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.3._source._doc_count: 2 } + - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - match: { hits.hits.3._source.metricset: pod } - match: { hits.hits.3._source.@timestamp: 2021-04-28T19:00:00.000Z } - length: { hits.hits.3._source.k8s.pod.latency.counts: 4 } - - match: { hits.hits.3._source.k8s.pod.latency.counts.0: 4 } - - match: { hits.hits.3._source.k8s.pod.latency.counts.1: 5 } - - match: { hits.hits.3._source.k8s.pod.latency.counts.2: 4 } - - match: { hits.hits.3._source.k8s.pod.latency.counts.3: 13 } + - match: { hits.hits.3._source.k8s.pod.latency.counts.0: 7 } + - match: { hits.hits.3._source.k8s.pod.latency.counts.1: 15 } + - match: { hits.hits.3._source.k8s.pod.latency.counts.2: 10 } + - match: { hits.hits.3._source.k8s.pod.latency.counts.3: 10 } - length: { hits.hits.3._source.k8s.pod.latency.values: 4 } - match: { hits.hits.3._source.k8s.pod.latency.values.0: 1.0 } - - match: { hits.hits.3._source.k8s.pod.latency.values.1: 10.0 } - - match: { hits.hits.3._source.k8s.pod.latency.values.2: 100.0 } - - match: { hits.hits.3._source.k8s.pod.latency.values.3: 1000.0 } + - match: { hits.hits.3._source.k8s.pod.latency.values.1: 2.0 } + - match: { hits.hits.3._source.k8s.pod.latency.values.2: 5.0 } + - match: { hits.hits.3._source.k8s.pod.latency.values.3: 10.0 } --- "Downsample date_nanos timestamp field using custom format": @@ -1228,8 +1229,8 @@ setup: --- "Downsample object field": - skip: - version: " - 8.7.99" - reason: "Bug fixed in version 8.8.0" + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.downsample: @@ -1250,54 +1251,54 @@ setup: - length: { hits.hits: 4 } - match: { hits.hits.0._source._doc_count: 2 } - - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - match: { hits.hits.0._source.metricset: pod } - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.0._source.k8s.pod.name: "cat" } - - match: { hits.hits.0._source.k8s.pod.value.min: 10.0 } - - match: { hits.hits.0._source.k8s.pod.value.max: 20.0 } - - match: { hits.hits.0._source.k8s.pod.value.sum: 30.0 } - - match: { hits.hits.0._source.k8s.pod.agent.id: "first" } - - match: { hits.hits.0._source.k8s.pod.agent.version: "2.0.4" } + - match: { hits.hits.0._source.k8s.pod.name: "dog" } + - match: { hits.hits.0._source.k8s.pod.value.min: 9.0 } + - match: { hits.hits.0._source.k8s.pod.value.max: 16.0 } + - match: { hits.hits.0._source.k8s.pod.value.sum: 25.0 } + - match: { hits.hits.0._source.k8s.pod.agent.id: "second" } + - match: { hits.hits.0._source.k8s.pod.agent.version: "2.1.7" } - match: { hits.hits.1._source._doc_count: 2 } - - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + - match: { hits.hits.1._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - match: { hits.hits.1._source.metricset: pod } - - match: { hits.hits.1._source.@timestamp: "2021-04-28T20:00:00.000Z" } - - match: { hits.hits.1._source.k8s.pod.name: "cat" } - - match: { hits.hits.1._source.k8s.pod.value.min: 12.0 } - - match: { hits.hits.1._source.k8s.pod.value.max: 15.0 } - - match: { hits.hits.1._source.k8s.pod.value.sum: 27.0 } - - match: { hits.hits.1._source.k8s.pod.agent.id: "first" } - - match: { hits.hits.1._source.k8s.pod.agent.version: "2.0.4" } + - match: { hits.hits.1._source.@timestamp: "2021-04-28T19:00:00.000Z" } + - match: { hits.hits.1._source.k8s.pod.name: "dog" } + - match: { hits.hits.1._source.k8s.pod.value.min: 17.0 } + - match: { hits.hits.1._source.k8s.pod.value.max: 25.0 } + - match: { hits.hits.1._source.k8s.pod.value.sum: 42.0 } + - match: { hits.hits.1._source.k8s.pod.agent.id: "second" } + - match: { hits.hits.1._source.k8s.pod.agent.version: "2.1.7" } - match: { hits.hits.2._source._doc_count: 2 } - - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - match: { hits.hits.2._source.metricset: pod } - match: { hits.hits.2._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.2._source.k8s.pod.name: "dog" } - - match: { hits.hits.2._source.k8s.pod.value.min: 9.0 } - - match: { hits.hits.2._source.k8s.pod.value.max: 16.0 } - - match: { hits.hits.2._source.k8s.pod.value.sum: 25.0 } - - match: { hits.hits.2._source.k8s.pod.agent.id: "second" } - - match: { hits.hits.2._source.k8s.pod.agent.version: "2.1.7" } + - match: { hits.hits.2._source.k8s.pod.name: "cat" } + - match: { hits.hits.2._source.k8s.pod.value.min: 10.0 } + - match: { hits.hits.2._source.k8s.pod.value.max: 20.0 } + - match: { hits.hits.2._source.k8s.pod.value.sum: 30.0 } + - match: { hits.hits.2._source.k8s.pod.agent.id: "first" } + - match: { hits.hits.2._source.k8s.pod.agent.version: "2.0.4" } - match: { hits.hits.3._source._doc_count: 2 } - - match: { hits.hits.3._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - match: { hits.hits.3._source.metricset: pod } - - match: { hits.hits.3._source.@timestamp: "2021-04-28T19:00:00.000Z" } - - match: { hits.hits.3._source.k8s.pod.name: "dog" } - - match: { hits.hits.3._source.k8s.pod.value.min: 17.0 } - - match: { hits.hits.3._source.k8s.pod.value.max: 25.0 } - - match: { hits.hits.3._source.k8s.pod.value.sum: 42.0 } - - match: { hits.hits.3._source.k8s.pod.agent.id: "second" } - - match: { hits.hits.3._source.k8s.pod.agent.version: "2.1.7" } + - match: { hits.hits.3._source.@timestamp: "2021-04-28T20:00:00.000Z" } + - match: { hits.hits.3._source.k8s.pod.name: "cat" } + - match: { hits.hits.3._source.k8s.pod.value.min: 12.0 } + - match: { hits.hits.3._source.k8s.pod.value.max: 15.0 } + - match: { hits.hits.3._source.k8s.pod.value.sum: 27.0 } + - match: { hits.hits.3._source.k8s.pod.agent.id: "first" } + - match: { hits.hits.3._source.k8s.pod.agent.version: "2.0.4" } --- "Downsample empty and missing labels": - skip: - version: " - 8.6.99" - reason: "Downsampling GA-ed in 8.7.0" + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.downsample: @@ -1317,17 +1318,17 @@ setup: - length: { hits.hits: 3 } - - match: { hits.hits.0._source._doc_count: 4 } - - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.0._source.metricset: pod } - - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.0._source.k8s.pod.name: "cat" } - - match: { hits.hits.0._source.k8s.pod.value.min: 10.0 } - - match: { hits.hits.0._source.k8s.pod.value.max: 40.0 } - - match: { hits.hits.0._source.k8s.pod.value.sum: 100.0 } - - match: { hits.hits.0._source.k8s.pod.value.value_count: 4 } - - match: { hits.hits.0._source.k8s.pod.label: "abc" } - - match: { hits.hits.0._source.k8s.pod.unmapped: "abc" } + - match: { hits.hits.2._source._doc_count: 4 } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + - match: { hits.hits.2._source.metricset: pod } + - match: { hits.hits.2._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.2._source.k8s.pod.name: "cat" } + - match: { hits.hits.2._source.k8s.pod.value.min: 10.0 } + - match: { hits.hits.2._source.k8s.pod.value.max: 40.0 } + - match: { hits.hits.2._source.k8s.pod.value.sum: 100.0 } + - match: { hits.hits.2._source.k8s.pod.value.value_count: 4 } + - match: { hits.hits.2._source.k8s.pod.label: "abc" } + - match: { hits.hits.2._source.k8s.pod.unmapped: "abc" } - match: { hits.hits.1._source._doc_count: 4 } - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e9597ab } @@ -1341,24 +1342,24 @@ setup: - match: { hits.hits.1._source.k8s.pod.label: null } - match: { hits.hits.1._source.k8s.pod.unmapped: null } - - match: { hits.hits.2._source._doc_count: 4 } - - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.2._source.metricset: pod } - - match: { hits.hits.2._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.2._source.k8s.pod.name: "dog" } - - match: { hits.hits.2._source.k8s.pod.value.min: 10.0 } - - match: { hits.hits.2._source.k8s.pod.value.max: 40.0 } - - match: { hits.hits.2._source.k8s.pod.value.sum: 100.0 } - - match: { hits.hits.2._source.k8s.pod.value.value_count: 4 } - - match: { hits.hits.2._source.k8s.pod.label: "xyz" } - - match: { hits.hits.2._source.k8s.pod.unmapped: "xyz" } + - match: { hits.hits.0._source._doc_count: 4 } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.0._source.metricset: pod } + - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.0._source.k8s.pod.name: "dog" } + - match: { hits.hits.0._source.k8s.pod.value.min: 10.0 } + - match: { hits.hits.0._source.k8s.pod.value.max: 40.0 } + - match: { hits.hits.0._source.k8s.pod.value.sum: 100.0 } + - match: { hits.hits.0._source.k8s.pod.value.value_count: 4 } + - match: { hits.hits.0._source.k8s.pod.label: "xyz" } + - match: { hits.hits.0._source.k8s.pod.unmapped: "xyz" } --- "Downsample label with ignore_above": - skip: - version: " - 8.7.99" - reason: "Downsample of time series index without metric allowed from version 8.8.0" + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -1441,37 +1442,37 @@ setup: - match: { hits.hits.0._source._doc_count: 2 } - match: { hits.hits.0._source.metricset: pod } - - match: { hits.hits.0._source.k8s.pod.name: fox } + - match: { hits.hits.0._source.k8s.pod.name: dog } - match: { hits.hits.0._source.k8s.pod.value: 20 } - - match: { hits.hits.0._source.k8s.pod.uid: 7393ef8e-489c-11ee-be56-0242ac120002 } - - match: { hits.hits.0._source.k8s.pod.label: bar } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.0._source.k8s.pod.label: foo } - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } - match: { hits.hits.1._source._doc_count: 2 } - match: { hits.hits.1._source.metricset: pod } - - match: { hits.hits.1._source.k8s.pod.name: cat } + - match: { hits.hits.1._source.k8s.pod.name: fox } - match: { hits.hits.1._source.k8s.pod.value: 20 } - - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - # NOTE: when downsampling a label field we propagate the last (most-recent timestamp-wise) non-null value, - # ignoring/skipping null values. Here the last document has a value that hits ignore_above ("foofoo") and, - # as a result, we propagate the value of the previous document ("foo") - - match: { hits.hits.1._source.k8s.pod.label: foo } + - match: { hits.hits.1._source.k8s.pod.uid: 7393ef8e-489c-11ee-be56-0242ac120002 } + - match: { hits.hits.1._source.k8s.pod.label: bar } - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:00:00.000Z } - match: { hits.hits.2._source._doc_count: 2 } - match: { hits.hits.2._source.metricset: pod } - - match: { hits.hits.2._source.k8s.pod.name: cow } + - match: { hits.hits.2._source.k8s.pod.name: cat } - match: { hits.hits.2._source.k8s.pod.value: 20 } - - match: { hits.hits.2._source.k8s.pod.uid: a81ef23a-489c-11ee-be56-0242ac120005 } - - match: { hits.hits.2._source.k8s.pod.label: null } + - match: { hits.hits.2._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } + # NOTE: when downsampling a label field we propagate the last (most-recent timestamp-wise) non-null value, + # ignoring/skipping null values. Here the last document has a value that hits ignore_above ("foofoo") and, + # as a result, we propagate the value of the previous document ("foo") + - match: { hits.hits.2._source.k8s.pod.label: foo } - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:00:00.000Z } - match: { hits.hits.3._source._doc_count: 2 } - match: { hits.hits.3._source.metricset: pod } - - match: { hits.hits.3._source.k8s.pod.name: dog } + - match: { hits.hits.3._source.k8s.pod.name: cow } - match: { hits.hits.3._source.k8s.pod.value: 20 } - - match: { hits.hits.3._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.3._source.k8s.pod.label: foo } + - match: { hits.hits.3._source.k8s.pod.uid: a81ef23a-489c-11ee-be56-0242ac120005 } + - match: { hits.hits.3._source.k8s.pod.label: null } - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:00:00.000Z } - do: From 0dff470b49fc13ddb83abff8dd20b5d27492fb42 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Nov 2023 16:46:51 +0100 Subject: [PATCH 039/125] fix: no way to get tsid from dimensions anymore --- .../bucket/TimeSeriesAggregationsIT.java | 227 ------------------ .../test/downsample/30_date_histogram.yml | 32 +-- .../test/downsample/40_runtime_fields.yml | 60 ++--- 3 files changed, 46 insertions(+), 273 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java index 2050ce20b1aee..cbd06b1b70020 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java @@ -15,55 +15,28 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.aggregations.AggregationIntegTestCase; -import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries; import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.RangeQueryBuilder; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.Aggregator; -import org.elasticsearch.search.aggregations.PipelineAggregatorBuilders; -import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; -import org.elasticsearch.search.aggregations.bucket.global.Global; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.metrics.CompensatedSum; -import org.elasticsearch.search.aggregations.metrics.Stats; -import org.elasticsearch.search.aggregations.metrics.Sum; -import org.elasticsearch.search.aggregations.pipeline.SimpleValue; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; -import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; -import static org.elasticsearch.search.aggregations.AggregationBuilders.global; -import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; -import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; -import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.search.aggregations.AggregationBuilders.topHits; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse; -import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.lessThan; -import static org.hamcrest.Matchers.lessThanOrEqualTo; @ESIntegTestCase.SuiteScopeTestCase public class TimeSeriesAggregationsIT extends AggregationIntegTestCase { @@ -175,206 +148,6 @@ public void setupSuiteScopeCluster() throws Exception { indexRandom(true, false, docs); } - public void testStandAloneTimeSeriesAgg() { - assertNoFailuresAndResponse(prepareSearch("index").setSize(0).addAggregation(timeSeries("by_ts")), response -> { - Aggregations aggregations = response.getAggregations(); - assertNotNull(aggregations); - InternalTimeSeries timeSeries = aggregations.get("by_ts"); - assertThat( - timeSeries.getBuckets().stream().map(MultiBucketsAggregation.Bucket::getKey).collect(Collectors.toSet()), - equalTo(data.keySet()) - ); - for (InternalTimeSeries.Bucket bucket : timeSeries.getBuckets()) { - @SuppressWarnings("unchecked") - Map key = (Map) bucket.getKey(); - assertThat((long) data.get(key).size(), equalTo(bucket.getDocCount())); - } - }); - } - - public void testTimeSeriesGroupedByADimension() { - String groupBy = "dim_" + randomIntBetween(0, numberOfDimensions - 1); - assertNoFailuresAndResponse( - prepareSearch("index").setSize(0) - .addAggregation( - terms("by_dim").field(groupBy) - .size(data.size()) - .collectMode(randomFrom(Aggregator.SubAggCollectionMode.values())) - .subAggregation(timeSeries("by_ts")) - ), - response -> { - Aggregations aggregations = response.getAggregations(); - assertNotNull(aggregations); - Terms terms = aggregations.get("by_dim"); - Set> keys = new HashSet<>(); - for (Terms.Bucket term : terms.getBuckets()) { - InternalTimeSeries timeSeries = term.getAggregations().get("by_ts"); - for (InternalTimeSeries.Bucket bucket : timeSeries.getBuckets()) { - @SuppressWarnings("unchecked") - Map key = (Map) bucket.getKey(); - assertThat((long) data.get(key).size(), equalTo(bucket.getDocCount())); - assertTrue("key is not unique", keys.add(key)); - assertThat( - "time series doesn't contain dimensions we grouped by", - key.get(groupBy), - equalTo(term.getKeyAsString()) - ); - } - } - assertThat(keys, equalTo(data.keySet())); - } - ); - } - - public void testTimeSeriesGroupedByDateHistogram() { - DateHistogramInterval fixedInterval = DateHistogramInterval.days(randomIntBetween(10, 100)); - assertNoFailuresAndResponse( - prepareSearch("index").setSize(0) - .addAggregation( - dateHistogram("by_time").field("@timestamp") - .fixedInterval(fixedInterval) - .subAggregation(timeSeries("by_ts").subAggregation(stats("timestamp").field("@timestamp"))) - ), - response -> { - Aggregations aggregations = response.getAggregations(); - assertNotNull(aggregations); - Histogram histogram = aggregations.get("by_time"); - Map, Long> keys = new HashMap<>(); - for (Histogram.Bucket interval : histogram.getBuckets()) { - long intervalStart = ((ZonedDateTime) interval.getKey()).toEpochSecond() * 1000; - long intervalEnd = intervalStart + fixedInterval.estimateMillis(); - InternalTimeSeries timeSeries = interval.getAggregations().get("by_ts"); - for (InternalTimeSeries.Bucket bucket : timeSeries.getBuckets()) { - @SuppressWarnings("unchecked") - Map key = (Map) bucket.getKey(); - keys.compute(key, (k, v) -> (v == null ? 0 : v) + bucket.getDocCount()); - assertThat(bucket.getDocCount(), lessThanOrEqualTo((long) data.get(key).size())); - Stats stats = bucket.getAggregations().get("timestamp"); - long minTimestamp = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(stats.getMinAsString()); - long maxTimestamp = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(stats.getMaxAsString()); - assertThat(minTimestamp, greaterThanOrEqualTo(intervalStart)); - assertThat(maxTimestamp, lessThan(intervalEnd)); - } - } - assertThat(keys.keySet(), equalTo(data.keySet())); - for (Map.Entry, Long> entry : keys.entrySet()) { - assertThat(entry.getValue(), equalTo((long) data.get(entry.getKey()).size())); - } - } - ); - } - - public void testStandAloneTimeSeriesAggWithDimFilter() { - boolean include = randomBoolean(); - int dim = randomIntBetween(0, numberOfDimensions - 1); - String val = randomFrom(dimensions[dim]); - QueryBuilder queryBuilder = QueryBuilders.termQuery("dim_" + dim, val); - if (include == false) { - queryBuilder = QueryBuilders.boolQuery().mustNot(queryBuilder); - } - assertNoFailuresAndResponse( - prepareSearch("index").setQuery(queryBuilder).setSize(0).addAggregation(timeSeries("by_ts")), - response -> { - Aggregations aggregations = response.getAggregations(); - assertNotNull(aggregations); - InternalTimeSeries timeSeries = aggregations.get("by_ts"); - Map, Map>> filteredData = dataFilteredByDimension("dim_" + dim, val, include); - assertThat( - timeSeries.getBuckets().stream().map(MultiBucketsAggregation.Bucket::getKey).collect(Collectors.toSet()), - equalTo(filteredData.keySet()) - ); - for (InternalTimeSeries.Bucket bucket : timeSeries.getBuckets()) { - @SuppressWarnings("unchecked") - Map key = (Map) bucket.getKey(); - assertThat(bucket.getDocCount(), equalTo((long) filteredData.get(key).size())); - } - } - ); - } - - public void testStandAloneTimeSeriesAggWithGlobalAggregation() { - boolean include = randomBoolean(); - int dim = randomIntBetween(0, numberOfDimensions - 1); - int metric = randomIntBetween(0, numberOfMetrics - 1); - String val = randomFrom(dimensions[dim]); - QueryBuilder queryBuilder = QueryBuilders.termQuery("dim_" + dim, val); - if (include == false) { - queryBuilder = QueryBuilders.boolQuery().mustNot(queryBuilder); - } - assertNoFailuresAndResponse( - prepareSearch("index").setQuery(queryBuilder) - .setSize(0) - .addAggregation(timeSeries("by_ts").subAggregation(sum("filter_sum").field("metric_" + metric))) - .addAggregation(global("everything").subAggregation(sum("all_sum").field("metric_" + metric))) - .addAggregation(PipelineAggregatorBuilders.sumBucket("total_filter_sum", "by_ts>filter_sum")), - response -> { - Aggregations aggregations = response.getAggregations(); - assertNotNull(aggregations); - InternalTimeSeries timeSeries = aggregations.get("by_ts"); - Map, Map>> filteredData = dataFilteredByDimension("dim_" + dim, val, include); - assertThat( - timeSeries.getBuckets().stream().map(MultiBucketsAggregation.Bucket::getKey).collect(Collectors.toSet()), - equalTo(filteredData.keySet()) - ); - for (InternalTimeSeries.Bucket bucket : timeSeries.getBuckets()) { - @SuppressWarnings("unchecked") - Map key = (Map) bucket.getKey(); - assertThat(bucket.getDocCount(), equalTo((long) filteredData.get(key).size())); - } - SimpleValue obj = aggregations.get("total_filter_sum"); - assertThat(obj.value(), closeTo(sumByMetric(filteredData, "metric_" + metric), obj.value() * 0.0001)); - - Global global = aggregations.get("everything"); - Sum allSum = global.getAggregations().get("all_sum"); - assertThat(allSum.value(), closeTo(sumByMetric(data, "metric_" + metric), allSum.value() * 0.0001)); - } - ); - - ElasticsearchException e = expectThrows( - ElasticsearchException.class, - () -> prepareSearch("index").setQuery(QueryBuilders.termQuery("dim_" + dim, val)) - .setSize(0) - .addAggregation(global("everything").subAggregation(timeSeries("by_ts"))) - .get() - ); - assertThat(e.getRootCause().getMessage(), containsString("Time series aggregations cannot be used inside global aggregation.")); - } - - public void testStandAloneTimeSeriesAggWithMetricFilter() { - boolean above = randomBoolean(); - int metric = randomIntBetween(0, numberOfMetrics - 1); - double val = randomDoubleBetween(0, 100000, true); - RangeQueryBuilder queryBuilder = QueryBuilders.rangeQuery("metric_" + metric); - if (above) { - queryBuilder.gt(val); - } else { - queryBuilder.lte(val); - } - assertNoFailuresAndResponse( - prepareSearch("index").setQuery(queryBuilder).setSize(0).addAggregation(timeSeries("by_ts")), - response -> { - Aggregations aggregations = response.getAggregations(); - assertNotNull(aggregations); - InternalTimeSeries timeSeries = aggregations.get("by_ts"); - Map, Map>> filteredData = dataFilteredByMetric( - data, - "metric_" + metric, - val, - above - ); - assertThat( - timeSeries.getBuckets().stream().map(MultiBucketsAggregation.Bucket::getKey).collect(Collectors.toSet()), - equalTo(filteredData.keySet()) - ); - for (InternalTimeSeries.Bucket bucket : timeSeries.getBuckets()) { - @SuppressWarnings("unchecked") - Map key = (Map) bucket.getKey(); - assertThat(bucket.getDocCount(), equalTo((long) filteredData.get(key).size())); - } - } - ); - } - public void testRetrievingHits() { Map.Entry filterMetric = randomMetricAndValue(data); double lowerVal = filterMetric.getValue() - randomDoubleBetween(0, 100000, true); diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/30_date_histogram.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/30_date_histogram.yml index b7f3ec7b8f384..1513d6814918a 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/30_date_histogram.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/30_date_histogram.yml @@ -74,37 +74,37 @@ setup: - match: { hits.hits.0._index: "test-downsample" } - match: { hits.hits.0._source._doc_count: 1 } - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.0._source.uid: "001" } - - close_to: { hits.hits.0._source.total_memory_used.min: { value: 106780.0, error: 0.00001 } } - - close_to: { hits.hits.0._source.total_memory_used.max: { value: 106780.0, error: 0.00001 } } - - close_to: { hits.hits.0._source.total_memory_used.sum: { value: 106780.0, error: 0.00001 } } + - match: { hits.hits.0._source.uid: "003" } + - close_to: { hits.hits.0._source.total_memory_used.min: { value: 109009.0, error: 0.00001 } } + - close_to: { hits.hits.0._source.total_memory_used.max: { value: 109009.0, error: 0.00001 } } + - close_to: { hits.hits.0._source.total_memory_used.sum: { value: 109009.0, error: 0.00001 } } - match: { hits.hits.0._source.total_memory_used.value_count: 1 } - match: { hits.hits.1._index: "test-downsample" } - match: { hits.hits.1._source._doc_count: 1 } - match: { hits.hits.1._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.1._source.uid: "002" } - - close_to: { hits.hits.1._source.total_memory_used.min: { value: 110450.0, error: 0.00001 } } - - close_to: { hits.hits.1._source.total_memory_used.max: { value: 110450.0, error: 0.00001 } } - - close_to: { hits.hits.1._source.total_memory_used.sum: { value: 110450.0, error: 0.00001 } } + - match: { hits.hits.1._source.uid: "004" } + - close_to: { hits.hits.1._source.total_memory_used.min: { value: 120770.0, error: 0.00001 } } + - close_to: { hits.hits.1._source.total_memory_used.max: { value: 120770.0, error: 0.00001 } } + - close_to: { hits.hits.1._source.total_memory_used.sum: { value: 120770.0, error: 0.00001 } } - match: { hits.hits.1._source.total_memory_used.value_count: 1 } - match: { hits.hits.2._index: "test-downsample" } - match: { hits.hits.2._source._doc_count: 1 } - match: { hits.hits.2._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.2._source.uid: "003" } - - close_to: { hits.hits.2._source.total_memory_used.min: { value: 109009.0, error: 0.00001 } } - - close_to: { hits.hits.2._source.total_memory_used.max: { value: 109009.0, error: 0.00001 } } - - close_to: { hits.hits.2._source.total_memory_used.sum: { value: 109009.0, error: 0.00001 } } + - match: { hits.hits.2._source.uid: "002" } + - close_to: { hits.hits.2._source.total_memory_used.min: { value: 110450.0, error: 0.00001 } } + - close_to: { hits.hits.2._source.total_memory_used.max: { value: 110450.0, error: 0.00001 } } + - close_to: { hits.hits.2._source.total_memory_used.sum: { value: 110450.0, error: 0.00001 } } - match: { hits.hits.2._source.total_memory_used.value_count: 1 } - match: { hits.hits.3._index: "test-downsample" } - match: { hits.hits.3._source._doc_count: 1 } - match: { hits.hits.3._source.@timestamp: "2021-04-28T18:00:00.000Z" } - - match: { hits.hits.3._source.uid: "004" } - - close_to: { hits.hits.3._source.total_memory_used.min: { value: 120770.0, error: 0.00001 } } - - close_to: { hits.hits.3._source.total_memory_used.max: { value: 120770.0, error: 0.00001 } } - - close_to: { hits.hits.3._source.total_memory_used.sum: { value: 120770.0, error: 0.00001 } } + - match: { hits.hits.3._source.uid: "001" } + - close_to: { hits.hits.3._source.total_memory_used.min: { value: 106780.0, error: 0.00001 } } + - close_to: { hits.hits.3._source.total_memory_used.max: { value: 106780.0, error: 0.00001 } } + - close_to: { hits.hits.3._source.total_memory_used.sum: { value: 106780.0, error: 0.00001 } } - match: { hits.hits.3._source.total_memory_used.value_count: 1 } # date histogram aggregation with calendar interval on rollup index not supported diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml index 06d74494e89c7..0cd94cc56eb77 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml @@ -1,8 +1,8 @@ --- "Runtime fields accessing metric fields in downsample target index": - skip: - version: " - 8.4.99" - reason: "downsample introduced in 8.5.0" + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 features: close_to - do: @@ -140,31 +140,31 @@ - length: { hits.hits: 4 } - - close_to: { hits.hits.0.fields.received_kb.0: { value: 783343.5488, error: 0.0001 } } - - close_to: { hits.hits.0.fields.sent_kb.0: { value: 1954908.8779, error: 0.0001 } } - - close_to: { hits.hits.0.fields.tx_kb.0: { value: 1958181.5957, error: 0.0001 } } - - close_to: { hits.hits.0.fields.rx_kb.0: { value: 783333.7832, error: 0.0001 } } + - close_to: { hits.hits.0.fields.received_kb.0: { value: 518142.3935, error: 0.0001 } } + - close_to: { hits.hits.0.fields.sent_kb.0: { value: 1400935.4472, error: 0.0001 } } + - close_to: { hits.hits.0.fields.tx_kb.0: { value: 1400955.0009, error: 0.0001 } } + - close_to: { hits.hits.0.fields.rx_kb.0: { value: 518164.1484, error: 0.0001 } } - - close_to: { hits.hits.1.fields.received_kb.0: { value: 783377.7343, error: 0.0001 } } - - close_to: { hits.hits.1.fields.sent_kb.0: { value: 1955339.7343, error: 0.0001 } } - - close_to: { hits.hits.1.fields.tx_kb.0: { value: 1965738.4785, error: 0.0001 } } - - close_to: { hits.hits.1.fields.rx_kb.0: { value: 784849.3369, error: 0.0001 } } + - close_to: { hits.hits.1.fields.received_kb.0: { value: 518186.5039, error: 0.0001 } } + - close_to: { hits.hits.1.fields.sent_kb.0: { value: 1400988.2822, error: 0.0001 } } + - close_to: { hits.hits.1.fields.tx_kb.0: { value: 1400971.9453, error: 0.0001 } } + - close_to: { hits.hits.1.fields.rx_kb.0: { value: 518169.4443, error: 0.0001 } } - - close_to: { hits.hits.2.fields.received_kb.0: { value: 518142.3935, error: 0.0001 } } - - close_to: { hits.hits.2.fields.sent_kb.0: { value: 1400935.4472, error: 0.0001 } } - - close_to: { hits.hits.2.fields.tx_kb.0: { value: 1400955.0009, error: 0.0001 } } - - close_to: { hits.hits.2.fields.rx_kb.0: { value: 518164.1484, error: 0.0001 } } + - close_to: { hits.hits.2.fields.received_kb.0: { value: 783343.5488, error: 0.0001 } } + - close_to: { hits.hits.2.fields.sent_kb.0: { value: 1954908.8779, error: 0.0001 } } + - close_to: { hits.hits.2.fields.tx_kb.0: { value: 1958181.5957, error: 0.0001 } } + - close_to: { hits.hits.2.fields.rx_kb.0: { value: 783333.7832, error: 0.0001 } } - - close_to: { hits.hits.3.fields.received_kb.0: { value: 518186.5039, error: 0.0001 } } - - close_to: { hits.hits.3.fields.sent_kb.0: { value: 1400988.2822, error: 0.0001 } } - - close_to: { hits.hits.3.fields.tx_kb.0: { value: 1400971.9453, error: 0.0001 } } - - close_to: { hits.hits.3.fields.rx_kb.0: { value: 518169.4443, error: 0.0001 } } + - close_to: { hits.hits.3.fields.received_kb.0: { value: 783377.7343, error: 0.0001 } } + - close_to: { hits.hits.3.fields.sent_kb.0: { value: 1955339.7343, error: 0.0001 } } + - close_to: { hits.hits.3.fields.tx_kb.0: { value: 1965738.4785, error: 0.0001 } } + - close_to: { hits.hits.3.fields.rx_kb.0: { value: 784849.3369, error: 0.0001 } } --- "Runtime field accessing dimension fields in downsample target index": - skip: - version: " - 8.4.99" - reason: "downsample introduced in 8.5.0" + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -276,16 +276,16 @@ - length: { hits.hits: 4 } - - match: { hits.hits.0.fields.metricset_tag.0: "pod-AAA" } - - match: { hits.hits.1.fields.metricset_tag.0: "pod-AAB" } - - match: { hits.hits.2.fields.metricset_tag.0: "pod-AAC" } - - match: { hits.hits.3.fields.metricset_tag.0: "pod-AAD" } + - match: { hits.hits.0.fields.metricset_tag.0: "pod-AAD" } + - match: { hits.hits.1.fields.metricset_tag.0: "pod-AAC" } + - match: { hits.hits.2.fields.metricset_tag.0: "pod-AAB" } + - match: { hits.hits.3.fields.metricset_tag.0: "pod-AAA" } --- "Runtime field accessing label fields in downsample target index": - skip: - version: " - 8.4.99" - reason: "downsample introduced in 8.5.0" + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.create: @@ -398,7 +398,7 @@ - length: { hits.hits: 4 } - - match: { hits.hits.0.fields.labels.0: "AAA-10" } - - match: { hits.hits.1.fields.labels.0: "AAB-110" } - - match: { hits.hits.2.fields.labels.0: "AAC-11" } - - match: { hits.hits.3.fields.labels.0: "AAD-111" } + - match: { hits.hits.0.fields.labels.0: "AAC-11" } + - match: { hits.hits.1.fields.labels.0: "AAD-111" } + - match: { hits.hits.2.fields.labels.0: "AAA-10" } + - match: { hits.hits.3.fields.labels.0: "AAB-110" } From a1b9f17d3c9f6e677583ab1ef15f39ec1c2da998 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Nov 2023 16:57:33 +0100 Subject: [PATCH 040/125] fix: decrease indexing pressure in test to avoid error --- .../bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java index c4dc4c4f36c30..7793a687a4e74 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -49,7 +49,7 @@ public void setup() throws Exception { final XContentBuilder mapping = timeSeriesIndexMapping(); long startMillis = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); long endMillis = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-02T00:00:00Z"); - numberOfDocuments = randomIntBetween(1000, 2000); + numberOfDocuments = randomIntBetween(100, 200); final Iterator timestamps = getTimestamps(startMillis, endMillis, numberOfDocuments); // NOTE: use also the last (changing) dimension so to make sure documents are not indexed all in the same shard. final String[] routingDimensions = new String[] { "dim_0", "dim_" + (numberOfDimensions - 1) }; From 950942c0827eb42404a628c88654d29711f77e79 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Nov 2023 17:17:03 +0100 Subject: [PATCH 041/125] fix: parse after key string as a BytesRef --- .../bucket/composite/CompositeAggregationBuilder.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java index 98f27b0a0ce49..d215eb2e3b688 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java @@ -246,14 +246,13 @@ protected AggregatorFactory doBuild( Object obj = after.get(sourceName); if (configs[i].missingBucket() && obj == null) { values[i] = null; - } else if (obj instanceof Comparable c && configs[i].fieldType().getClass() != TimeSeriesIdFieldType.class) { + } else if (obj instanceof String && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { + values[i] = configs[i].format().parseBytesRef(obj); + } else if (obj instanceof Comparable c) { values[i] = c; } else if (obj instanceof Map && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { // If input is a _tsid map, encode the map to the _tsid BytesRef values[i] = configs[i].format().parseBytesRef(obj); - } else if (obj instanceof String && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { - // If input is a _tsid string, encode the string to the _tsid BytesRef - values[i] = configs[i].format().parseBytesRef(obj); } else { throw new IllegalArgumentException( "Invalid value for [after." From 8a626cace66fb9158967efa481fa110a97c05c1d Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Nov 2023 17:24:28 +0100 Subject: [PATCH 042/125] fix: downsampling test after tsid hashing --- .../test/downsample/10_basic.yml | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index 2ef436f61a3c9..6605aac85a017 100644 --- a/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -293,8 +293,8 @@ setup: --- "Downsample index": - skip: - version: " - 8.4.99" - reason: "Downsampling GA-ed in 8.7.0" + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: indices.downsample: @@ -313,24 +313,24 @@ setup: sort: [ "_tsid", "@timestamp" ] - length: { hits.hits: 4 } - - match: { hits.hits.0._source._doc_count: 2 } - - match: { hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.0._source.metricset: pod } - - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } - - match: { hits.hits.0._source.k8s.pod.multi-counter: 21 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 90 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 200 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 726 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 } - - match: { hits.hits.0._source.k8s.pod.network.tx.min: 2001818691 } - - match: { hits.hits.0._source.k8s.pod.network.tx.max: 2005177954 } - - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 } - - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.26" } - - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:35:00.000Z" } - - match: { hits.hits.0._source.k8s.pod.number_of_containers: 2 } - - match: { hits.hits.0._source.k8s.pod.tags: ["backend", "prod", "us-west1"] } - - match: { hits.hits.0._source.k8s.pod.values: [1, 1, 3] } - - is_true: hits.hits.0._source.k8s.pod.running + - match: { hits.hits.0._source._doc_count: 2 } + - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } + - match: { hits.hits.0._source.metricset: pod } + - match: { hits.hits.0._source.@timestamp: "2021-04-28T18:00:00.000Z" } + - match: { hits.hits.0._source.k8s.pod.multi-counter: 0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 100.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 102.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 607.0 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 } + - match: { hits.hits.0._source.k8s.pod.network.tx.min: 1434521831 } + - match: { hits.hits.0._source.k8s.pod.network.tx.max: 1434577921 } + - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 } + - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.56" } + - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:43:00.000Z" } + - match: { hits.hits.0._source.k8s.pod.number_of_containers: 1 } + - match: { hits.hits.0._source.k8s.pod.tags: [ "backend", "test", "us-west2" ] } + - match: { hits.hits.0._source.k8s.pod.values: [ 1, 1, 2 ] } + - is_false: hits.hits.0._source.k8s.pod.running # Assert downsample index settings - do: From 0b0d11409b3d1e7b40b102064739bd1ab86507a2 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Nov 2023 17:27:12 +0100 Subject: [PATCH 043/125] fix: possible NPE --- .../CompositeAggregationBuilder.java | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java index d215eb2e3b688..396c5c870a364 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java @@ -246,22 +246,24 @@ protected AggregatorFactory doBuild( Object obj = after.get(sourceName); if (configs[i].missingBucket() && obj == null) { values[i] = null; - } else if (obj instanceof String && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { - values[i] = configs[i].format().parseBytesRef(obj); - } else if (obj instanceof Comparable c) { - values[i] = c; - } else if (obj instanceof Map && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { - // If input is a _tsid map, encode the map to the _tsid BytesRef - values[i] = configs[i].format().parseBytesRef(obj); - } else { - throw new IllegalArgumentException( - "Invalid value for [after." - + sources.get(i).name() - + "], expected comparable, got [" - + (obj == null ? "null" : obj.getClass().getSimpleName()) - + "]" - ); - } + } else if (obj instanceof String + && configs[i].fieldType() != null + && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { + values[i] = configs[i].format().parseBytesRef(obj); + } else if (obj instanceof Comparable c) { + values[i] = c; + } else if (obj instanceof Map && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { + // If input is a _tsid map, encode the map to the _tsid BytesRef + values[i] = configs[i].format().parseBytesRef(obj); + } else { + throw new IllegalArgumentException( + "Invalid value for [after." + + sources.get(i).name() + + "], expected comparable, got [" + + (obj == null ? "null" : obj.getClass().getSimpleName()) + + "]" + ); + } } afterKey = new CompositeKey(values); } else { From 2417509adb044794335aef375e155bb18db33b92 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Nov 2023 18:55:19 +0100 Subject: [PATCH 044/125] fix: TimeSeriesIdFieldMapperTsidHashingTests --- ...meSeriesIdFieldMapperTsidHashingTests.java | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java index 502a8b5598c28..2d920d35a35d9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java @@ -84,8 +84,8 @@ public void testEnabledInTimeSeriesMode() throws Exception { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100).field("c", 500)); assertEquals( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - "30CO74tuRyatuMXEvNJvbgAAAAAAAAAAAAAAAAAAAADuDjjxPwT2Ol0UAzaqxX0Rpx75AkZHRdY" + "30CO74tuRyatuMXEvNJvbu4OOPE_BPY6XRQDNqrFfRGnHvkCRkdF1g", + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) ); } @@ -132,8 +132,8 @@ public void testStrings() throws IOException { b -> b.field("a", "foo").field("b", "bar").field("c", "baz").startObject("o").field("e", "bort").endObject() ); assertEquals( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - "S3oqtElkKwgW-mWA1nr9gAAAAAAAAAAAAAAAAAAAAACv9houtA-WtALkiErAWD5o8PqlkO2CPtw" + "S3oqtElkKwgW-mWA1nr9gK_2Gi60D5a0AuSISsBYPmjw-qWQ7YI-3A", + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()) ); } @@ -148,8 +148,8 @@ public void testUnicodeKeys() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field(fire, "hot").field(coffee, "good")); assertEquals( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - "A-nLhW-M69syDAiaaHLtCgAAAAAAAAAAAAAAAAAAAACejAXjDKlQ6oAVV6S1YhlDnYAgGZ4kqeg" + "A-nLhW-M69syDAiaaHLtCp6MBeMMqVDqgBVXpLViGUOdgCAZniSp6A", + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) ); } @@ -161,8 +161,8 @@ public void testKeywordTooLong() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); assertEquals( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - "FlVJQe85E3Mv4Ekz_gxSawAAAAAAAAAAAAAAAAAAAAC9ZDUoCO2eynkD-lZbEGc5huKIYw" + "FlVJQe85E3Mv4Ekz_gxSa71kNSgI7Z7KeQP6VlsQZzmG4ohj", + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) ); } @@ -175,8 +175,8 @@ public void testKeywordTooLongUtf8() throws IOException { String theWordLong = "長い"; ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); assertEquals( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - "FlVJQe85E3Mv4Ekz_gxSawAAAAAAAAAAAAAAAAAAAABM43f6n6D4bFDIodLIvFw5j_1Vew" + "FlVJQe85E3Mv4Ekz_gxSa0zjd_qfoPhsUMih0si8XDmP_VV7", + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) ); } @@ -217,8 +217,8 @@ public void testLong() throws IOException { b.startObject("o").field("e", 1234).endObject(); }); assertEquals( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJy91wa2A3yf66JQYgw9_3GPhrunFA" + "iI8WAECVhaC5BYgDlkz8OK2bjW9KuknL3XBrYDfJ_rolBiDD3_cY-Gu6cUA", + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()) ); } @@ -272,8 +272,8 @@ public void testInteger() throws IOException { b.startObject("o").field("e", Integer.MIN_VALUE).endObject(); }); assertEquals( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJy9XyEVAdPjbPnyFfNkppmgRrzARX" + "iI8WAECVhaC5BYgDlkz8OK2bjW9KuknL1fIRUB0-Ns-fIV82SmmaBGvMBFc", + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()) ); } @@ -331,8 +331,8 @@ public void testShort() throws IOException { b.startObject("o").field("e", Short.MIN_VALUE).endObject(); }); assertEquals( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJyyYAWY_JC1ss-P30niOYXA2x9kLq" + "iI8WAECVhaC5BYgDlkz8OK2bjW9KuknLJgBZj8kLWyz4_fSeI5hcDbH2Quo", + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()) ); } @@ -390,8 +390,8 @@ public void testByte() throws IOException { b.startObject("o").field("e", (int) Byte.MIN_VALUE).endObject(); }); assertEquals( - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), - "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAACtm41vSrpJyygrqocRYcbJqQJZlIyTzrY9S9Nt" + "iI8WAECVhaC5BYgDlkz8OK2bjW9KuknLKCuqhxFhxsmpAlmUjJPOtj1L020", + TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()) ); } @@ -449,8 +449,8 @@ public void testIp() throws IOException { b.startObject("o").field("e", "255.255.255.1").endObject(); }); assertEquals( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - "iI8WAECVhaC5BYgDlkz8OAAAAAAAAAAAAAAAAAAAAAALrOlWSrpJy7PzTuwgaqmbsw6Oz00Ir_8Cw9Xm" + "iI8WAECVhaC5BYgDlkz8OAus6VZKuknLs_NO7CBqqZuzDo7PTQiv_wLD1eY", + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) ); } @@ -486,13 +486,13 @@ public void testVeryLarge() throws IOException { } }); assertEquals( - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), - "QZ1K9tk9UIcpA826eU6H-QAAAAAAAAAAAAAAAAAAAACv9houWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWK" - + "lBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ip" - + "QZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqU" - + "GTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlB" - + "k1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk8VBv2" - + "rLPN9GYxF8e8uXojI" + "QZ1K9tk9UIcpA826eU6H-a_2Gi5YqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKl" + + "Bk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZN" + + "YqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKl" + + "Bk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZN" + + "YqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKl" + + "Bk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTxUG_ass830ZjEXx7y5eiMg", + TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) ); } @@ -628,7 +628,7 @@ public void testDifferentMetricNamesSameValues() throws IOException { double metricValue = randomDoubleBetween(10, 20, true); ParsedDocument doc1 = parseDocument(docMapper1, d -> d.field("a", "value").field("b", 10).field("m1", metricValue)); ParsedDocument doc2 = parseDocument(docMapper2, d -> d.field("a", "value").field("b", 10).field("m2", metricValue)); - assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); + assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); } /** From cbd99fdb4d864c5c5b859c467888e74d2e5924d4 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Nov 2023 19:29:44 +0100 Subject: [PATCH 045/125] fix: TsidExtractingIdFieldMapper --- .../mapper/TsidExtractingIdFieldMapper.java | 2 +- .../TsidExtractingIdFieldMapperTests.java | 582 ++++++++++++------ 2 files changed, 395 insertions(+), 189 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index de22880dc9c48..b51608ee9505b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -188,7 +188,7 @@ public String documentDescription(DocumentParserContext context) { StringBuilder description = new StringBuilder("a time series document"); IndexableField tsidField = context.doc().getField(TimeSeriesIdFieldMapper.NAME); if (tsidField != null) { - description.append(" with dimensions ").append(tsidDescription(tsidField)); + description.append(" with tsid ").append(tsidDescription(tsidField)); } IndexableField timestampField = context.doc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); if (timestampField != null) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index 320057e878f1c..199ff2e52f803 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -79,139 +79,273 @@ public static Iterable params() { */ // Dates - items.add(new TestCase("2022-01-01T01:00:00Z", "XsFI2ezm5OViFixWAAABfhMmioA", "{r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - })); - items.add(new TestCase("2022-01-01T01:00:01Z", "XsFI2ezm5OViFixWAAABfhMmjmg", "{r1=cat}", "2022-01-01T01:00:01.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:01Z"); - b.field("r1", "cat"); - })); - items.add(new TestCase("1970-01-01T00:00:00Z", "XsFI2ezm5OViFixWAAAAAAAAAAA", "{r1=cat}", "1970-01-01T00:00:00.000Z", b -> { - b.field("@timestamp", "1970-01-01T00:00:00Z"); - b.field("r1", "cat"); - })); - items.add(new TestCase("-9998-01-01T00:00:00Z", "XsFI2ezm5OViFixW__6oggRgGAA", "{r1=cat}", "-9998-01-01T00:00:00.000Z", b -> { - b.field("@timestamp", "-9998-01-01T00:00:00Z"); - b.field("r1", "cat"); - })); - items.add(new TestCase("9998-01-01T00:00:00Z", "XsFI2ezm5OViFixWAADmaSK9hAA", "{r1=cat}", "9998-01-01T00:00:00.000Z", b -> { - b.field("@timestamp", "9998-01-01T00:00:00Z"); - b.field("r1", "cat"); - })); + items.add( + new TestCase( + "2022-01-01T01:00:00Z", + "XsFI2ajcFfi45iV3AAABfhMmioA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + } + ) + ); + items.add( + new TestCase( + "2022-01-01T01:00:01Z", + "XsFI2ajcFfi45iV3AAABfhMmjmg", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "2022-01-01T01:00:01.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:01Z"); + b.field("r1", "cat"); + } + ) + ); + items.add( + new TestCase( + "1970-01-01T00:00:00Z", + "XsFI2ajcFfi45iV3AAAAAAAAAAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "1970-01-01T00:00:00.000Z", + b -> { + b.field("@timestamp", "1970-01-01T00:00:00Z"); + b.field("r1", "cat"); + } + ) + ); + items.add( + new TestCase( + "-9998-01-01T00:00:00Z", + "XsFI2ajcFfi45iV3__6oggRgGAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "-9998-01-01T00:00:00.000Z", + b -> { + b.field("@timestamp", "-9998-01-01T00:00:00Z"); + b.field("r1", "cat"); + } + ) + ); + items.add( + new TestCase( + "9998-01-01T00:00:00Z", + "XsFI2ajcFfi45iV3AADmaSK9hAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "9998-01-01T00:00:00.000Z", + b -> { + b.field("@timestamp", "9998-01-01T00:00:00Z"); + b.field("r1", "cat"); + } + ) + ); // routing keywords - items.add(new TestCase("r1", "XsFI2ezm5OViFixWAAABfhMmioA", "{r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("k1", (String) null); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("L1", (Long) null); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("i1", (Integer) null); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("s1", (Short) null); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("b1", (Byte) null); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("ip1", (String) null); - })); - items.add(new TestCase("r2", "1y-UzdYi98F0UVRiAAABfhMmioA", "{r2=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r2", "cat"); - })); - items.add(new TestCase("o.r3", "zh4dcftpIU55Ond-AAABfhMmioA", "{o.r3=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.startObject("o").field("r3", "cat").endObject(); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("o.r3", "cat"); - })); + items.add( + new TestCase( + "r1", + "XsFI2ajcFfi45iV3AAABfhMmioA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + } + ).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k1", (String) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("L1", (Long) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("i1", (Integer) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("s1", (Short) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("b1", (Byte) null); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("ip1", (String) null); + }) + ); + items.add( + new TestCase( + "r2", + "1y-UzR0iuE1-sOQpAAABfhMmioA", + "JNY_frTR9GmCbhXgK4Y8W44GlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r2", "cat"); + } + ) + ); + items.add( + new TestCase( + "o.r3", + "zh4dcS1h1gf2J5a8AAABfhMmioA", + "JEyfZsJIp3UNyfWG-4SjKFIGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o").field("r3", "cat").endObject(); + } + ).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + }) + ); // non-routing keyword - items.add(new TestCase("k1=dog", "XsFI2dL8sZeQhBgxAAABfhMmioA", "{k1=dog, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("k1", "dog"); - })); - items.add(new TestCase("k1=pumpkin", "XsFI2VlD6_SkSo4MAAABfhMmioA", "{k1=pumpkin, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("k1", "pumpkin"); - })); - items.add(new TestCase("k1=empty string", "XsFI2aBA6UgrxLRqAAABfhMmioA", "{k1=, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("k1", ""); - })); - items.add(new TestCase("k2", "XsFI2W2e5Ycw0o5_AAABfhMmioA", "{k2=dog, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("k2", "dog"); - })); - items.add(new TestCase("o.k3", "XsFI2ZAfOI6DMQhFAAABfhMmioA", "{o.k3=dog, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.startObject("o").field("k3", "dog").endObject(); - })); - items.add(new TestCase("o.r3", "zh4dcbFtT1qHtjl8AAABfhMmioA", "{o.k3=dog, o.r3=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.startObject("o"); - { - b.field("r3", "cat"); - b.field("k3", "dog"); - } - b.endObject(); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("o.r3", "cat"); - b.startObject("o").field("k3", "dog").endObject(); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.startObject("o").field("r3", "cat").endObject(); - b.field("o.k3", "dog"); - }).and(b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("o.r3", "cat"); - b.field("o.k3", "dog"); - })); - - // long - items.add(new TestCase("L1=1", "XsFI2eGMFOYjW7LLAAABfhMmioA", "{L1=1, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("L1", 1); - })); items.add( - new TestCase("L1=min", "XsFI2f9V0yuDfkRWAAABfhMmioA", "{L1=" + Long.MIN_VALUE + ", r1=cat}", "2022-01-01T01:00:00.000Z", b -> { + new TestCase( + "k1=dog", + "XsFI2SrEiVgZlSsYAAABfhMmioA", + "KJQKpjU9U63jhh-eNJ1f8bipyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k1", "dog"); + } + ) + ); + items.add( + new TestCase( + "k1=pumpkin", + "XsFI2W8GX8-0QcFxAAABfhMmioA", + "KJQKpjU9U63jhh-eNJ1f8bibzw1JBpU_-VsHjSz5HC1yy_swPEM1iGoAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k1", "pumpkin"); + } + ) + ); + items.add( + new TestCase( + "k1=empty string", + "XsFI2cna58i6D-Q6AAABfhMmioA", + "KJQKpjU9U63jhh-eNJ1f8bhaCD7uBpU_-SWGG0Uv9tZ1mLO2gi9rC1IAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k1", ""); + } + ) + ); + items.add( + new TestCase( + "k2", + "XsFI2VqlzAuv-06kAAABfhMmioA", + "KB9H-tGrL_UzqMcqXcgBtzypyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("k2", "dog"); + } + ) + ); + items.add( + new TestCase( + "o.k3", + "XsFI2S_VhridAKDUAAABfhMmioA", + "KGXATwN7ISd1_EycFRJ9h6qpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.startObject("o").field("k3", "dog").endObject(); + } + ) + ); + items.add( + new TestCase( + "o.r3", + "zh4dcUwfL7x__2oPAAABfhMmioA", + "KJaYZVZz8plfkEvvPBpi1EWpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o"); + { + b.field("r3", "cat"); + b.field("k3", "dog"); + } + b.endObject(); + } + ).and(b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("L1", Long.MIN_VALUE); + b.field("o.r3", "cat"); + b.startObject("o").field("k3", "dog").endObject(); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.startObject("o").field("r3", "cat").endObject(); + b.field("o.k3", "dog"); + }).and(b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("o.r3", "cat"); + b.field("o.k3", "dog"); }) ); - items.add(new TestCase("L2=1234", "XsFI2S8PYEBSm6QYAAABfhMmioA", "{L2=1234, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("L2", 1234); - })); + + // long + items.add( + new TestCase( + "L1=1", + "XsFI2fIe53BtV9PCAAABfhMmioA", + "KI4kVxcCLIMM2_VQGD575d-tm41vBpU_-TUExUU_bL3Puq_EBgIaLacAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("L1", 1); + } + ) + ); + items.add( + new TestCase( + "L1=min", + "XsFI2Qhu7hy1RoXRAAABfhMmioA", + "KI4kVxcCLIMM2_VQGD575d8caJ3TBpU_-cLpg-VnCBnhYk33HZBle6EAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("L1", Long.MIN_VALUE); + } + ) + ); + items.add( + new TestCase( + "L2=1234", + "XsFI2QTrNu7TTpc-AAABfhMmioA", + "KI_1WxF60L0IczG5ftUCWdndcGtgBpU_-QfM2BaR0DMagIfw3TDu_mAAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("L2", 1234); + } + ) + ); items.add( new TestCase( "o.L3=max", - "zh4dcaI-57LdG7-cAAABfhMmioA", - "{o.L3=" + Long.MAX_VALUE + ", o.r3=cat}", + "zh4dcWBQI6THHqxoAAABfhMmioA", + "KN4a6QzKhzc3nwzNLuZkV51xxTOVBpU_-erUU1qSW4eJ0kP0RmAB9TEAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00.000Z"); @@ -238,16 +372,24 @@ public static Iterable params() { ); // int - items.add(new TestCase("i1=1", "XsFI2R3LiMZSeUGKAAABfhMmioA", "{i1=1, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("i1", 1); - })); + items.add( + new TestCase( + "i1=1", + "XsFI2UMS_RWRoHYjAAABfhMmioA", + "KLGFpvAV8QkWSmX54kXFMgitm41vBpU_-TUExUU_bL3Puq_EBgIaLacAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("i1", 1); + } + ) + ); items.add( new TestCase( "i1=min", - "XsFI2fC7DMEVFaU9AAABfhMmioA", - "{i1=" + Integer.MIN_VALUE + ", r1=cat}", + "XsFI2adlQM5ILoA1AAABfhMmioA", + "KLGFpvAV8QkWSmX54kXFMgjV8hFQBpU_-WG2MicRGWwJdBKWq2F4qy4AAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -256,16 +398,24 @@ public static Iterable params() { } ) ); - items.add(new TestCase("i2=1234", "XsFI2ZVte8HK90RJAAABfhMmioA", "{i2=1324, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("i2", 1324); - })); + items.add( + new TestCase( + "i2=1234", + "XsFI2bhxfB6J0kBFAAABfhMmioA", + "KJc4-5eN1uAlYuAknQQLUlxavn2sBpU_-UEXBjgaH1uYcbayrOhdgpcAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("i2", 1324); + } + ) + ); items.add( new TestCase( "o.i3=max", - "zh4dcQy_QJRCqIx7AAABfhMmioA", - "{o.i3=" + Integer.MAX_VALUE + ", o.r3=cat}", + "zh4dcelxKf19CbfdAAABfhMmioA", + "KKqnzPNBe8ObksSo8rNaIFPZPCcBBpU_-Rhd_U6Jn2pjQz2zpmBuJb4AAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -292,28 +442,50 @@ public static Iterable params() { ); // short - items.add(new TestCase("s1=1", "XsFI2axCr11Q93m7AAABfhMmioA", "{r1=cat, s1=1}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("s1", 1); - })); items.add( - new TestCase("s1=min", "XsFI2Rbs9Ua9BH1wAAABfhMmioA", "{r1=cat, s1=" + Short.MIN_VALUE + "}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("s1", Short.MIN_VALUE); - }) + new TestCase( + "s1=1", + "XsFI2Y_y-8kD_BFeAAABfhMmioA", + "KFi_JDbvzWyAawmh8IEXedwGlT_5rZuNb-1ruHTTZhtsXRZpZRwWFocAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("s1", 1); + } + ) + ); + items.add( + new TestCase( + "s1=min", + "XsFI2WV8VNVnmPVNAAABfhMmioA", + "KFi_JDbvzWyAawmh8IEXedwGlT_5JgBZj9BSCms2_jgeFFhsmDlNFdMAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("s1", Short.MIN_VALUE); + } + ) + ); + items.add( + new TestCase( + "s2=1234", + "XsFI2VO8mUr-J5CpAAABfhMmioA", + "KKEQ2p3CkpMH61hNk_SuvI0GlT_53XBrYP5TPdmCR-vREPnt20e9f9wAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("s2", 1234); + } + ) ); - items.add(new TestCase("s2=1234", "XsFI2SBKaLBqXMBYAAABfhMmioA", "{r1=cat, s2=1234}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("s2", 1234); - })); items.add( new TestCase( "o.s3=max", - "zh4dcYIFo98LQWs4AAABfhMmioA", - "{o.r3=cat, o.s3=" + Short.MAX_VALUE + "}", + "zh4dcQKh6K11zWeuAAABfhMmioA", + "KKVMoT_-GS95fvIBtR7XK9oGlT_5Dme9-H3sen0WZ7leJpCj7-vXau4AAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -340,28 +512,50 @@ public static Iterable params() { ); // byte - items.add(new TestCase("b1=1", "XsFI2dDrcWaf3zDPAAABfhMmioA", "{b1=1, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("b1", 1); - })); items.add( - new TestCase("b1=min", "XsFI2cTzLrNqHtxnAAABfhMmioA", "{b1=" + Byte.MIN_VALUE + ", r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("b1", Byte.MIN_VALUE); - }) + new TestCase( + "b1=1", + "XsFI2dKxqgT5JDQfAAABfhMmioA", + "KGPAUhTjWOsRfDmYp3SUELatm41vBpU_-TUExUU_bL3Puq_EBgIaLacAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("b1", 1); + } + ) + ); + items.add( + new TestCase( + "b1=min", + "XsFI2d_PD--DgUvoAAABfhMmioA", + "KGPAUhTjWOsRfDmYp3SUELYoK6qHBpU_-d8HkZFJ3aL2ZV1lgHAjT1gAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("b1", Byte.MIN_VALUE); + } + ) + ); + items.add( + new TestCase( + "b2=12", + "XsFI2aqX5QjiuhsEAAABfhMmioA", + "KA58oUMzXeX1V5rh51Ste0K5K9vPBpU_-Wn8JQplO-x3CgoslYO5VksAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("b2", 12); + } + ) ); - items.add(new TestCase("b2=12", "XsFI2Sb77VB9AswjAAABfhMmioA", "{b2=12, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("b2", 12); - })); items.add( new TestCase( "o.s3=max", - "zh4dcfFauKzj6lgxAAABfhMmioA", - "{o.b3=" + Byte.MAX_VALUE + ", o.r3=cat}", + "zh4dccJ4YtN_21XHAAABfhMmioA", + "KIwZH-StJBobjk9tCV-0OgjKmuwGBpU_-Sd-SdnoH3sbfKLgse-briEAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -389,22 +583,34 @@ public static Iterable params() { // ip items.add( - new TestCase("ip1=192.168.0.1", "XsFI2dJ1cyrrjNa2AAABfhMmioA", "{ip1=192.168.0.1, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("ip1", "192.168.0.1"); - }).and(b -> { + new TestCase( + "ip1=192.168.0.1", + "XsFI2T5km9raIz_rAAABfhMmioA", + "KNj6cLPRNEkqdjfOPIbg0wULrOlWBpU_-efWDsz6B6AnnwbZ7GeeocEAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("ip1", "192.168.0.1"); + } + ).and(b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); b.field("r1", "cat"); b.field("ip1", "::ffff:c0a8:1"); }) ); items.add( - new TestCase("ip1=12.12.45.254", "XsFI2ZUAcRxOwhHKAAABfhMmioA", "{ip1=12.12.45.254, r1=cat}", "2022-01-01T01:00:00.000Z", b -> { - b.field("@timestamp", "2022-01-01T01:00:00Z"); - b.field("r1", "cat"); - b.field("ip1", "12.12.45.254"); - }).and(b -> { + new TestCase( + "ip1=12.12.45.254", + "XsFI2QWfEH_e_6wIAAABfhMmioA", + "KNj6cLPRNEkqdjfOPIbg0wVhJ08TBpU_-bANzLhvKPczlle7Pq0z8QwAAAAAAAAA", + "2022-01-01T01:00:00.000Z", + b -> { + b.field("@timestamp", "2022-01-01T01:00:00Z"); + b.field("r1", "cat"); + b.field("ip1", "12.12.45.254"); + } + ).and(b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); b.field("r1", "cat"); b.field("ip1", "::ffff:c0c:2dfe"); @@ -413,8 +619,8 @@ public static Iterable params() { items.add( new TestCase( "ip2=FE80:CD00:0000:0CDE:1257:0000:211E:729C", - "XsFI2XTGWAekP_oGAAABfhMmioA", - "{ip2=fe80:cd00:0:cde:1257:0:211e:729c, r1=cat}", + "XsFI2WrrLHr1O4iQAAABfhMmioA", + "KNDo3zGxO9HfN9XYJwKw2Z20h-WsBpU_-f4dSOLGSRlL1hoY2mgERuoAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -426,8 +632,8 @@ public static Iterable params() { items.add( new TestCase( "o.ip3=2001:db8:85a3:8d3:1319:8a2e:370:7348", - "zh4dcU_FSGP9GuHjAAABfhMmioA", - "{o.ip3=2001:db8:85a3:8d3:1319:8a2e:370:7348, o.r3=cat}", + "zh4dca7d-9aKOS1MAAABfhMmioA", + "KLXDcBBWJAjgJvjSdF_EJwraAQUzBpU_-ba6HZsIyKnGcbmc3KRLlmIAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -457,8 +663,8 @@ public static Iterable params() { items.add( new TestCase( "huge", - "WZKJR1Fi00B8Afr-AAABfhMmioA", - "{k1=" + huge + ", k2=" + huge.substring(0, 191) + "...}", + "WZKJR_dECvXBSl3xAAABfhMmioA", + "LIe18i0rRU_Bt9vB82F46LaS9mrUkvZq1K_2Gi7UEFMhFwNXrLA_H8TLpUr4AAAAAAAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -603,11 +809,11 @@ public void testSourceDescription() throws IOException { IndexableField tsid = d.rootDoc().getField(TimeSeriesIdFieldMapper.NAME); assertThat( TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(tsid)), - equalTo("a time series document with dimensions " + testCase.expectedTsid) + equalTo("a time series document with tsid " + testCase.expectedTsid) ); assertThat( TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(tsid, timestamp)), - equalTo("a time series document with dimensions " + testCase.expectedTsid + " at [" + testCase.expectedTimestamp + "]") + equalTo("a time series document with tsid " + testCase.expectedTsid + " at [" + testCase.expectedTimestamp + "]") ); } From fa321d879906e3b7922e39fd106096ab7027489f Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Nov 2023 19:36:36 +0100 Subject: [PATCH 046/125] fix: sime other yaml tests --- .../test/aggregations/time_series.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 7f003ba31964e..b645d20aa31e2 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -76,7 +76,7 @@ setup: - match: { hits.total.value: 1 } - length: { aggregations: 1 } - - match: { aggregations.ts.buckets.0.key: { _tsid: "QxzdeAQNbMXbXqwEXmfHBQAAAAAAAAAAAAAAAAAAAACv9hou4__k9oZfeombF5dLPwkqUA" } } + - match: { aggregations.ts.buckets.0.key: "JEMc3XgEDWzF216sBF5nxwWv9hou4__k9oZfeombF5dLPwkqUA" } - match: { aggregations.ts.buckets.0.doc_count: 1 } --- @@ -126,7 +126,7 @@ setup: size: 1 - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: { _tsid: "QxzdeAQNbMXbXqwEXmfHBQAAAAAAAAAAAAAAAAAAAACKf_6WZzMy4zeMjdIBoQC3xHGkjg" } } + - match: { aggregations.ts.buckets.0.key: "JEMc3XgEDWzF216sBF5nxwWKf_6WZzMy4zeMjdIBoQC3xHGkjg" } - do: search: @@ -144,9 +144,9 @@ setup: size: 3 - length: { aggregations.ts.buckets: 3 } - - match: { aggregations.ts.buckets.0.key: { _tsid: "QxzdeAQNbMXbXqwEXmfHBQAAAAAAAAAAAAAAAAAAAACKf_6WZzMy4zeMjdIBoQC3xHGkjg" } } - - match: { aggregations.ts.buckets.1.key: { _tsid: "QxzdeAQNbMXbXqwEXmfHBQAAAAAAAAAAAAAAAAAAAACv9hou4__k9oZfeombF5dLPwkqUA" } } - - match: { aggregations.ts.buckets.2.key: { _tsid: "QxzdeAQNbMXbXqwEXmfHBQAAAAAAAAAAAAAAAAAAAAC5AZB2-M4K_VYiHrKDjUNX3zHVkw" } } + - match: { aggregations.ts.buckets.0.key: "JEMc3XgEDWzF216sBF5nxwWKf_6WZzMy4zeMjdIBoQC3xHGkjg" } + - match: { aggregations.ts.buckets.1.key: "JEMc3XgEDWzF216sBF5nxwWv9hou4__k9oZfeombF5dLPwkqUA" } + - match: { aggregations.ts.buckets.2.key: "JEMc3XgEDWzF216sBF5nxwW5AZB2-M4K_VYiHrKDjUNX3zHVkw" } --- "Score test filter some": @@ -311,8 +311,8 @@ setup: --- "Number for keyword routing field": - skip: - version: " - 8.10.99" - reason: "Fix in 8.11" + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 - do: bulk: @@ -341,5 +341,5 @@ setup: - match: { hits.total.value: 2 } - length: { aggregations: 1 } - - match: { aggregations.ts.buckets.0.key: { "_tsid": "QxzdeAQNbMXbXqwEXmfHBQAAAAAAAAAAAAAAAAAAAAC6KPQ_9D8hCF3DOnGNi7WMdDeQmw" } } + - match: { aggregations.ts.buckets.0.key: "JEMc3XgEDWzF216sBF5nxwW6KPQ_9D8hCF3DOnGNi7WMdDeQmw" } - match: { aggregations.ts.buckets.0.doc_count: 1 } From e046bbb7a2e68dedef0cf60211fd7e25e919326a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Nov 2023 10:26:07 +0100 Subject: [PATCH 047/125] fix: a few more tests --- .../index/mapper/DocumentMapperTests.java | 14 ++++-- .../composite/CompositeAggregatorTests.java | 50 +++++++++++++------ 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index f0458add93c78..3f35852a0f3f0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -329,9 +329,13 @@ public void testEmptyDocumentMapper() { public void testTooManyDimensionFields() { int max; Settings settings; + String dimensionErrorSuffix = ""; if (randomBoolean()) { - max = 21; // By default no more than 21 dimensions per document are supported - settings = getIndexSettings(); + max = 1000; // By default no more than 1000 dimensions per document are supported + settings = Settings.builder() + .put(getIndexSettings()) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), max) + .build(); } else { max = between(1, 10000); settings = Settings.builder() @@ -339,6 +343,7 @@ public void testTooManyDimensionFields() { .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), max) .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), max + 1) .build(); + dimensionErrorSuffix = "dimension "; } IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> createMapperService(settings, mapping(b -> { for (int i = 0; i <= max; i++) { @@ -348,7 +353,10 @@ public void testTooManyDimensionFields() { .endObject(); } }))); - assertThat(e.getMessage(), containsString("Limit of total dimension fields [" + max + "] has been exceeded")); + assertThat( + e.getMessage(), + containsString(String.format("Limit of total %sfields [" + max + "] has been exceeded", dimensionErrorSuffix)) + ); } public void testDeeplyNestedMapping() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index b3488f31ebcb4..c36175ae45d66 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -2873,15 +2873,15 @@ public void testWithTsid() throws Exception { () -> new CompositeAggregationBuilder("name", Collections.singletonList(new TermsValuesSourceBuilder("tsid").field("_tsid"))), (InternalComposite result) -> { assertEquals(4, result.getBuckets().size()); - assertEquals("{tsid={dim1=foo, dim2=200}}", result.afterKey().toString()); + assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA}", result.afterKey().toString()); - assertEquals("{tsid={dim1=bar, dim2=100}}", result.getBuckets().get(0).getKeyAsString()); + assertEquals("{tsid=AgRkaW0xcwNiYXIEZGltMmwAAAAAAAAAZA}", result.getBuckets().get(0).getKeyAsString()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); - assertEquals("{tsid={dim1=bar, dim2=200}}", result.getBuckets().get(1).getKeyAsString()); + assertEquals("{tsid=AgRkaW0xcwNiYXIEZGltMmwAAAAAAAAAyA}", result.getBuckets().get(1).getKeyAsString()); assertEquals(1L, result.getBuckets().get(1).getDocCount()); - assertEquals("{tsid={dim1=foo, dim2=100}}", result.getBuckets().get(2).getKeyAsString()); + assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAZA}", result.getBuckets().get(2).getKeyAsString()); assertEquals(2L, result.getBuckets().get(2).getDocCount()); - assertEquals("{tsid={dim1=foo, dim2=200}}", result.getBuckets().get(3).getKeyAsString()); + assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA}", result.getBuckets().get(3).getKeyAsString()); assertEquals(1L, result.getBuckets().get(3).getDocCount()); } ); @@ -2893,10 +2893,10 @@ public void testWithTsid() throws Exception { ); }, (InternalComposite result) -> { assertEquals(2, result.getBuckets().size()); - assertEquals("{tsid={dim1=foo, dim2=200}}", result.afterKey().toString()); - assertEquals("{tsid={dim1=foo, dim2=100}}", result.getBuckets().get(0).getKeyAsString()); + assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA}", result.afterKey().toString()); + assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAZA}", result.getBuckets().get(0).getKeyAsString()); assertEquals(2L, result.getBuckets().get(0).getDocCount()); - assertEquals("{tsid={dim1=foo, dim2=200}}", result.getBuckets().get(1).getKeyAsString()); + assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA}", result.getBuckets().get(1).getKeyAsString()); assertEquals(1L, result.getBuckets().get(1).getDocCount()); }); } @@ -2925,15 +2925,27 @@ public void testWithTsidAndDateHistogram() throws IOException { ), (InternalComposite result) -> { assertEquals(4, result.getBuckets().size()); - assertEquals("{tsid={dim1=foo, dim2=200}, date_histo=1634688000000}", result.afterKey().toString()); + assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA, date_histo=1634688000000}", result.afterKey().toString()); - assertEquals("{tsid={dim1=foo, dim2=100}, date_histo=1632096000000}", result.getBuckets().get(0).getKeyAsString()); + assertEquals( + "{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAZA, date_histo=1632096000000}", + result.getBuckets().get(0).getKeyAsString() + ); assertEquals(1L, result.getBuckets().get(0).getDocCount()); - assertEquals("{tsid={dim1=foo, dim2=100}, date_histo=1634688000000}", result.getBuckets().get(1).getKeyAsString()); + assertEquals( + "{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAZA, date_histo=1634688000000}", + result.getBuckets().get(1).getKeyAsString() + ); assertEquals(1L, result.getBuckets().get(1).getDocCount()); - assertEquals("{tsid={dim1=foo, dim2=200}, date_histo=1632096000000}", result.getBuckets().get(2).getKeyAsString()); + assertEquals( + "{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA, date_histo=1632096000000}", + result.getBuckets().get(2).getKeyAsString() + ); assertEquals(1L, result.getBuckets().get(2).getDocCount()); - assertEquals("{tsid={dim1=foo, dim2=200}, date_histo=1634688000000}", result.getBuckets().get(3).getKeyAsString()); + assertEquals( + "{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA, date_histo=1634688000000}", + result.getBuckets().get(3).getKeyAsString() + ); assertEquals(2L, result.getBuckets().get(3).getDocCount()); } ); @@ -2950,10 +2962,16 @@ public void testWithTsidAndDateHistogram() throws IOException { ).aggregateAfter(createAfterKey("tsid", createTsid(Map.of("dim1", "foo", "dim2", 100)), "date_histo", 1634688000000L)), (InternalComposite result) -> { assertEquals(2, result.getBuckets().size()); - assertEquals("{tsid={dim1=foo, dim2=200}, date_histo=1634688000000}", result.afterKey().toString()); - assertEquals("{tsid={dim1=foo, dim2=200}, date_histo=1632096000000}", result.getBuckets().get(0).getKeyAsString()); + assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA, date_histo=1634688000000}", result.afterKey().toString()); + assertEquals( + "{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA, date_histo=1632096000000}", + result.getBuckets().get(0).getKeyAsString() + ); assertEquals(1L, result.getBuckets().get(0).getDocCount()); - assertEquals("{tsid={dim1=foo, dim2=200}, date_histo=1634688000000}", result.getBuckets().get(1).getKeyAsString()); + assertEquals( + "{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA, date_histo=1634688000000}", + result.getBuckets().get(1).getKeyAsString() + ); assertEquals(2L, result.getBuckets().get(1).getDocCount()); } ); From 41663e3f72d6a1b0c4990f43a86db04ef59150f9 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Nov 2023 10:48:13 +0100 Subject: [PATCH 048/125] fix: remove metrics --- .../org/elasticsearch/index/mapper/DocumentFields.java | 7 ------- .../index/mapper/TimeSeriesIdFieldMapper.java | 8 -------- 2 files changed, 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentFields.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentFields.java index 434a74b80e8b7..d79dbe4f95a68 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentFields.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentFields.java @@ -44,8 +44,6 @@ default void addKeywordDimension(String fieldName, String value) { void addUnsignedLongDimension(String fieldName, long value); - void addMetric(String fieldName); - /** * Makes sure that each dimension only appears on time. */ @@ -78,11 +76,6 @@ public void addUnsignedLongDimension(String fieldName, long value) { add(fieldName); } - @Override - public void addMetric(String fieldName) { - add(fieldName); - } - private void add(String fieldName) { boolean isNew = names.add(fieldName); if (false == isNew) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 62a5d937ffd05..7e2f810ce7d95 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -40,7 +40,6 @@ import java.util.Base64; import java.util.Collections; import java.util.Comparator; -import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -180,7 +179,6 @@ private record DimensionDataHolder(BytesRef name, BytesReference value) {} * to build the _tsid field for the document. */ private final SortedSet dimensions = new TreeSet<>(Comparator.comparing(o -> o.name)); - private final Set metrics = new TreeSet<>(); /** * Builds the routing. Used for building {@code _id}. If null then skipped. */ @@ -211,7 +209,6 @@ public BytesReference withoutHash() throws IOException { * with the following pattern: * * hash128(catenate(dimension field names)) + - * hash128(catenate(metric field names)) + * foreach(dimension field value, limit = MAX_DIMENSIONS) { hash32(dimension field value) } + * hash128(catenate(dimension field values)) * @@ -323,11 +320,6 @@ public void addUnsignedLongDimension(String fieldName, long value) { } } - @Override - public void addMetric(String fieldName) { - metrics.add(fieldName); - } - private void add(String fieldName, BytesReference encoded) throws IOException { final DimensionDataHolder dimension = new DimensionDataHolder(new BytesRef(fieldName), encoded); if (dimensions.contains(dimension)) { From b12e5e67def1fb7b1589a8f6834b0c0171026451 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Nov 2023 11:01:05 +0100 Subject: [PATCH 049/125] fix: undo interface and method renaming --- .../timeseries/InternalTimeSeriesTests.java | 2 +- .../InternalTimeSeriesTsidHashingTests.java | 2 +- .../timeseries/TimeSeriesAggregatorTests.java | 4 +-- .../TimeSeriesAggregatorTsidHashingTests.java | 4 +-- .../org/elasticsearch/index/IndexMode.java | 10 +++---- ...entFields.java => DocumentDimensions.java} | 28 +++++++++---------- .../index/mapper/DocumentParserContext.java | 14 +++++----- .../index/mapper/IpFieldMapper.java | 2 +- .../index/mapper/KeywordFieldMapper.java | 2 +- .../index/mapper/NumberFieldMapper.java | 4 +-- .../index/mapper/TimeSeriesIdFieldMapper.java | 14 +++++----- .../flattened/FlattenedFieldParser.java | 2 +- .../elasticsearch/search/DocValueFormat.java | 8 +++--- .../index/mapper/IdLoaderTests.java | 8 +++--- .../mapper/IdLoaderTsidHashingTests.java | 8 +++--- .../search/DocValueFormatTests.java | 6 ++-- .../rate/TimeSeriesRateAggregatorTests.java | 2 +- .../unsignedlong/UnsignedLongFieldMapper.java | 4 +-- .../aggregations/GeoLineAggregatorTests.java | 2 +- 19 files changed, 61 insertions(+), 65 deletions(-) rename server/src/main/java/org/elasticsearch/index/mapper/{DocumentFields.java => DocumentDimensions.java} (66%) diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java index e1674e881f8eb..cc65968ec77ed 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java @@ -50,7 +50,7 @@ private List randomBuckets(boolean keyed, InternalAggregations a long docCount = randomLongBetween(0, Long.MAX_VALUE / (20L * numberOfBuckets)); var builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); for (var entry : keys.get(j).entrySet()) { - builder.addKeywordDimension(entry.getKey(), (String) entry.getValue()); + builder.addString(entry.getKey(), (String) entry.getValue()); } try { var key = builder.withHash().toBytesRef(); diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTsidHashingTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTsidHashingTests.java index 5db7b52ea1578..1a68ce11f71eb 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTsidHashingTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTsidHashingTests.java @@ -50,7 +50,7 @@ private List randomBuckets(boolean keyed, InternalAggregations a long docCount = randomLongBetween(0, Long.MAX_VALUE / (20L * numberOfBuckets)); var builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); for (var entry : keys.get(j).entrySet()) { - builder.addKeywordDimension(entry.getKey(), (String) entry.getValue()); + builder.addString(entry.getKey(), (String) entry.getValue()); } try { var key = builder.withHash().toBytesRef(); diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index 96c4eb0d475b1..7767ec9595950 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -96,9 +96,9 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens final TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(null); for (int i = 0; i < dimensions.length; i += 2) { if (dimensions[i + 1] instanceof Number n) { - builder.addLongDimension(dimensions[i].toString(), n.longValue()); + builder.addLong(dimensions[i].toString(), n.longValue()); } else { - builder.addKeywordDimension(dimensions[i].toString(), dimensions[i + 1].toString()); + builder.addString(dimensions[i].toString(), dimensions[i + 1].toString()); } } for (int i = 0; i < metrics.length; i += 2) { diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java index 22d8deea00cc9..f49ac048a1181 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java @@ -125,9 +125,9 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens final TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(null); for (int i = 0; i < dimensions.length; i += 2) { if (dimensions[i + 1] instanceof Number n) { - builder.addLongDimension(dimensions[i].toString(), n.longValue()); + builder.addLong(dimensions[i].toString(), n.longValue()); } else { - builder.addKeywordDimension(dimensions[i].toString(), dimensions[i + 1].toString()); + builder.addString(dimensions[i].toString(), dimensions[i + 1].toString()); } } for (int i = 0; i < metrics.length; i += 2) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 118433f6e18ef..05afc14e0f0cd 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -17,7 +17,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.DocumentFields; +import org.elasticsearch.index.mapper.DocumentDimensions; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperService; @@ -103,8 +103,8 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public DocumentFields buildDocumentDimensions(IndexSettings settings) { - return new DocumentFields.OnlySingleValueAllowed(); + public DocumentDimensions buildDocumentDimensions(IndexSettings settings) { + return new DocumentDimensions.OnlySingleValueAllowed(); } @Override @@ -196,7 +196,7 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { } @Override - public DocumentFields buildDocumentDimensions(IndexSettings settings) { + public DocumentDimensions buildDocumentDimensions(IndexSettings settings) { IndexRouting.ExtractFromSource routing = (IndexRouting.ExtractFromSource) settings.getIndexRouting(); return new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routing.builder()); } @@ -325,7 +325,7 @@ public String getName() { /** * How {@code time_series_dimension} fields are handled by indices in this mode. */ - public abstract DocumentFields buildDocumentDimensions(IndexSettings settings); + public abstract DocumentDimensions buildDocumentDimensions(IndexSettings settings); /** * @return Whether timestamps should be validated for being withing the time range of an index. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentFields.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java similarity index 66% rename from server/src/main/java/org/elasticsearch/index/mapper/DocumentFields.java rename to server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java index d79dbe4f95a68..fafa7f7a9cb12 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentFields.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java @@ -18,12 +18,12 @@ /** * Collects dimensions from documents. */ -public interface DocumentFields { +public interface DocumentDimensions { /** * Build an index's DocumentDimensions using its settings */ - static DocumentFields fromIndexSettings(IndexSettings indexSettings) { + static DocumentDimensions fromIndexSettings(IndexSettings indexSettings) { return indexSettings.getMode().buildDocumentDimensions(indexSettings); } @@ -32,47 +32,47 @@ static DocumentFields fromIndexSettings(IndexSettings indexSettings) { * value is already computed in some cases when we want to collect * dimensions, so we can save re-computing the UTF-8 encoding. */ - void addKeywordDimension(String fieldName, BytesRef utf8Value); + void addString(String fieldName, BytesRef utf8Value); - default void addKeywordDimension(String fieldName, String value) { - addKeywordDimension(fieldName, new BytesRef(value)); + default void addString(String fieldName, String value) { + addString(fieldName, new BytesRef(value)); } - void addIpDimension(String fieldName, InetAddress value); + void addIp(String fieldName, InetAddress value); - void addLongDimension(String fieldName, long value); + void addLong(String fieldName, long value); - void addUnsignedLongDimension(String fieldName, long value); + void addUnsignedLong(String fieldName, long value); /** * Makes sure that each dimension only appears on time. */ - class OnlySingleValueAllowed implements DocumentFields { + class OnlySingleValueAllowed implements DocumentDimensions { private final Set names = new HashSet<>(); @Override - public void addKeywordDimension(String fieldName, BytesRef value) { + public void addString(String fieldName, BytesRef value) { add(fieldName); } // Override to skip the UTF-8 conversion that happens in the default implementation @Override - public void addKeywordDimension(String fieldName, String value) { + public void addString(String fieldName, String value) { add(fieldName); } @Override - public void addIpDimension(String fieldName, InetAddress value) { + public void addIp(String fieldName, InetAddress value) { add(fieldName); } @Override - public void addLongDimension(String fieldName, long value) { + public void addLong(String fieldName, long value) { add(fieldName); } @Override - public void addUnsignedLongDimension(String fieldName, long value) { + public void addUnsignedLong(String fieldName, long value) { add(fieldName); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 82b12a76dded0..f47b392115f81 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -88,7 +88,7 @@ protected void addDoc(LuceneDocument doc) { private final Set newFieldsSeen; private final Map dynamicObjectMappers; private final List dynamicRuntimeFields; - private final DocumentFields documentFields; + private final DocumentDimensions dimensions; private final ObjectMapper parent; private final ObjectMapper.Dynamic dynamic; private String id; @@ -109,7 +109,7 @@ private DocumentParserContext( String id, Field version, SeqNoFieldMapper.SequenceIDFields seqID, - DocumentFields documentFields, + DocumentDimensions dimensions, ObjectMapper parent, ObjectMapper.Dynamic dynamic, Set fieldsAppliedFromTemplates, @@ -126,7 +126,7 @@ private DocumentParserContext( this.id = id; this.version = version; this.seqID = seqID; - this.documentFields = documentFields; + this.dimensions = dimensions; this.parent = parent; this.dynamic = dynamic; this.fieldsAppliedFromTemplates = fieldsAppliedFromTemplates; @@ -146,7 +146,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, in.id, in.version, in.seqID, - in.documentFields, + in.dimensions, parent, dynamic, in.fieldsAppliedFromTemplates, @@ -173,7 +173,7 @@ protected DocumentParserContext( null, null, null, - DocumentFields.fromIndexSettings(mappingParserContext.getIndexSettings()), + DocumentDimensions.fromIndexSettings(mappingParserContext.getIndexSettings()), parent, dynamic, new HashSet<>(), @@ -524,8 +524,8 @@ public XContentParser parser() { /** * The collection of dimensions for this document. */ - public DocumentFields getDocumentFields() { - return documentFields; + public DocumentDimensions getDimensions() { + return dimensions; } public abstract ContentPath path(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 94b727818fb94..56a50c2dee0aa 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -555,7 +555,7 @@ private static InetAddress value(XContentParser parser, InetAddress nullValue) t private void indexValue(DocumentParserContext context, InetAddress address) { if (dimension) { - context.getDocumentFields().addIpDimension(fieldType().name(), address); + context.getDimensions().addIp(fieldType().name(), address); } if (indexed) { Field field = new InetAddressPoint(fieldType().name(), address); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index e16227a4f068d..caac5b7f3bfe0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -906,7 +906,7 @@ private void indexValue(DocumentParserContext context, String value) { final BytesRef binaryValue = new BytesRef(value); if (fieldType().isDimension()) { - context.getDocumentFields().addKeywordDimension(fieldType().name(), binaryValue); + context.getDimensions().addString(fieldType().name(), binaryValue); } // If the UTF8 encoding of the field value is bigger than the max length 32766, Lucene fill fail the indexing request and, to diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 901265b55c8a2..091e3c61764b0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1872,9 +1872,7 @@ public Number value(XContentParser parser) throws IllegalArgumentException, IOEx */ public void indexValue(DocumentParserContext context, Number numericValue) { if (dimension && numericValue != null) { - context.getDocumentFields().addLongDimension(fieldType().name(), numericValue.longValue()); - } else if (fieldType().getMetricType() != null) { - context.getDocumentFields().addMetric(fieldType().name()); + context.getDimensions().addLong(fieldType().name(), numericValue.longValue()); } fieldType().type.addFields(context.doc(), fieldType().name(), numericValue, indexed, hasDocValues, stored); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 7e2f810ce7d95..fd99483b2e8eb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -130,7 +130,7 @@ private TimeSeriesIdFieldMapper() { public void postParse(DocumentParserContext context) throws IOException { assert fieldType().isIndexed() == false; - final TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDocumentFields(); + final TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDimensions(); final BytesRef timeSeriesId = getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ID_HASHING) ? timeSeriesIdBuilder.withoutHash().toBytesRef() : timeSeriesIdBuilder.withHash().toBytesRef(); @@ -163,7 +163,7 @@ public static Object decodeTsid(StreamInput in) { } } - public static class TimeSeriesIdBuilder implements DocumentFields { + public static class TimeSeriesIdBuilder implements DocumentDimensions { private static final int SEED = 0; @@ -268,7 +268,7 @@ private int writeHash128(final MurmurHash3.Hash128 hash128, byte[] buffer, int t } @Override - public void addKeywordDimension(String fieldName, BytesRef utf8Value) { + public void addString(String fieldName, BytesRef utf8Value) { try (BytesStreamOutput out = new BytesStreamOutput()) { out.write((byte) 's'); /* @@ -288,12 +288,12 @@ public void addKeywordDimension(String fieldName, BytesRef utf8Value) { } @Override - public void addIpDimension(String fieldName, InetAddress value) { - addKeywordDimension(fieldName, NetworkAddress.format(value)); + public void addIp(String fieldName, InetAddress value) { + addString(fieldName, NetworkAddress.format(value)); } @Override - public void addLongDimension(String fieldName, long value) { + public void addLong(String fieldName, long value) { try (BytesStreamOutput out = new BytesStreamOutput()) { out.write((byte) 'l'); out.writeLong(value); @@ -304,7 +304,7 @@ public void addLongDimension(String fieldName, long value) { } @Override - public void addUnsignedLongDimension(String fieldName, long value) { + public void addUnsignedLong(String fieldName, long value) { try (BytesStreamOutput out = new BytesStreamOutput()) { Object ul = DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value); if (ul instanceof Long l) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java index 389a223ddc182..f09c6f8c036c8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java @@ -173,7 +173,7 @@ private void addField(DocumentParserContext context, ContentPath path, String cu final String keyedFieldName = FlattenedFieldParser.extractKey(bytesKeyedValue).utf8ToString(); if (fieldType.isDimension() && fieldType.dimensions().contains(keyedFieldName)) { final BytesRef keyedFieldValue = FlattenedFieldParser.extractValue(bytesKeyedValue); - context.getDocumentFields().addKeywordDimension(rootFieldName + "." + keyedFieldName, keyedFieldValue); + context.getDimensions().addString(rootFieldName + "." + keyedFieldName, keyedFieldValue); } } } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 18179a9603c0e..8dc218b7db68f 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -725,14 +725,14 @@ private BytesRef parseBytesRefMap(Object value) { Object v = entry.getValue(); if (v instanceof String s) { - builder.addKeywordDimension(f, s); + builder.addString(f, s); } else if (v instanceof Long l) { - builder.addLongDimension(f, l); + builder.addLong(f, l); } else if (v instanceof Integer i) { - builder.addLongDimension(f, i.longValue()); + builder.addLong(f, i.longValue()); } else if (v instanceof BigInteger ul) { long ll = UNSIGNED_LONG_SHIFTED.parseLong(ul.toString(), false, () -> 0L); - builder.addUnsignedLongDimension(f, ll); + builder.addUnsignedLong(f, ll); } else { throw new IllegalArgumentException("Unexpected value in tsid object [" + v + "]"); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index c94e0dd14ace0..1edf73eed8c00 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -228,10 +228,10 @@ private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter fields.add(new LongPoint(DataStreamTimestampFieldMapper.DEFAULT_PATH, doc.timestamp)); for (Dimension dimension : doc.dimensions) { if (dimension.value instanceof Number n) { - builder.addLongDimension(dimension.field, n.longValue()); + builder.addLong(dimension.field, n.longValue()); fields.add(new SortedNumericDocValuesField(dimension.field, ((Number) dimension.value).longValue())); } else { - builder.addKeywordDimension(dimension.field, dimension.value.toString()); + builder.addString(dimension.field, dimension.value.toString()); fields.add(new SortedSetDocValuesField(dimension.field, new BytesRef(dimension.value.toString()))); } } @@ -245,9 +245,9 @@ private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routingBuilder); for (Dimension dimension : doc.dimensions) { if (dimension.value instanceof Number n) { - timeSeriesIdBuilder.addLongDimension(dimension.field, n.longValue()); + timeSeriesIdBuilder.addLong(dimension.field, n.longValue()); } else { - timeSeriesIdBuilder.addKeywordDimension(dimension.field, dimension.value.toString()); + timeSeriesIdBuilder.addString(dimension.field, dimension.value.toString()); } } return TsidExtractingIdFieldMapper.createId( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java index 2070bc8d4dcf9..053432df6cc23 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java @@ -227,10 +227,10 @@ private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter fields.add(new LongPoint(DataStreamTimestampFieldMapper.DEFAULT_PATH, doc.timestamp)); for (Dimension dimension : doc.dimensions) { if (dimension.value instanceof Number n) { - builder.addLongDimension(dimension.field, n.longValue()); + builder.addLong(dimension.field, n.longValue()); fields.add(new SortedNumericDocValuesField(dimension.field, ((Number) dimension.value).longValue())); } else { - builder.addKeywordDimension(dimension.field, dimension.value.toString()); + builder.addString(dimension.field, dimension.value.toString()); fields.add(new SortedSetDocValuesField(dimension.field, new BytesRef(dimension.value.toString()))); } } @@ -244,9 +244,9 @@ private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routingBuilder); for (Dimension dimension : doc.dimensions) { if (dimension.value instanceof Number n) { - timeSeriesIdBuilder.addLongDimension(dimension.field, n.longValue()); + timeSeriesIdBuilder.addLong(dimension.field, n.longValue()); } else { - timeSeriesIdBuilder.addKeywordDimension(dimension.field, dimension.value.toString()); + timeSeriesIdBuilder.addString(dimension.field, dimension.value.toString()); } } return TsidExtractingIdFieldMapper.createId( diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index c50a42099b037..e2a25635fb335 100644 --- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -374,9 +374,9 @@ public void testParseZone() { public void testParseTsid() throws IOException { TimeSeriesIdBuilder timeSeriesIdBuilder = new TimeSeriesIdBuilder(null); - timeSeriesIdBuilder.addKeywordDimension("string", randomAlphaOfLength(10)); - timeSeriesIdBuilder.addLongDimension("long", randomLong()); - timeSeriesIdBuilder.addUnsignedLongDimension("ulong", randomLong()); + timeSeriesIdBuilder.addString("string", randomAlphaOfLength(10)); + timeSeriesIdBuilder.addLong("long", randomLong()); + timeSeriesIdBuilder.addUnsignedLong("ulong", randomLong()); BytesRef expected = timeSeriesIdBuilder.withHash().toBytesRef(); Object tsidFormat = DocValueFormat.TIME_SERIES_ID.format(expected); BytesRef actual = DocValueFormat.TIME_SERIES_ID.parseBytesRef(tsidFormat); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index 01440cf2a44f2..4b7688209b81f 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -166,7 +166,7 @@ private List docs(long startTimestamp, String dim, long... values) thr private static BytesReference tsid(String dim) throws IOException { TimeSeriesIdFieldMapper.TimeSeriesIdBuilder idBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); - idBuilder.addKeywordDimension("dim", dim); + idBuilder.addString("dim", dim); return idBuilder.withHash(); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index a795b149ec794..97ffd50d5b8c3 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -633,9 +633,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } if (dimension && numericValue != null) { - context.getDocumentFields().addUnsignedLongDimension(fieldType().name(), numericValue); - } else if (fieldType().getMetricType() != null) { - context.getDocumentFields().addMetric(fieldType().name()); + context.getDimensions().addUnsignedLong(fieldType().name(), numericValue); } List fields = new ArrayList<>(); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index b61dbe1e4ae3c..b367393be2498 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -803,7 +803,7 @@ private void assertGeoLine_TSDB( ArrayList timestamps = testData.timestampsForGroup(g); for (int i = 0; i < points.size(); i++) { final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); - builder.addKeywordDimension("group_id", testData.groups[g]); + builder.addString("group_id", testData.groups[g]); ArrayList fields = new ArrayList<>( Arrays.asList( new SortedDocValuesField("group_id", new BytesRef(testData.groups[g])), From 51ff8b527f109c3e765ca5b2cb5337b1938ea7c2 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Nov 2023 11:05:30 +0100 Subject: [PATCH 050/125] fix: just change ordering of docs --- .../org/elasticsearch/index/mapper/IdLoaderTests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index 1edf73eed8c00..6dd1bc799eb8b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -53,8 +53,8 @@ public void testSynthesizeIdSimple() throws Exception { long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs = List.of( - new Doc(startTime, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))), - new Doc(startTime + 1, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), + new Doc(startTime, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), + new Doc(startTime + 1, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))), new Doc(startTime + 2, List.of(new Dimension("dim1", "bbb"), new Dimension("dim2", "xxx"))) ); CheckedConsumer verify = indexReader -> { @@ -64,8 +64,8 @@ public void testSynthesizeIdSimple() throws Exception { var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 1, 2 }); // NOTE: time series data is ordered by (tsid, timestamp) assertThat(leaf.getId(0), equalTo(expectedId(routing, docs.get(2)))); - assertThat(leaf.getId(1), equalTo(expectedId(routing, docs.get(1)))); - assertThat(leaf.getId(2), equalTo(expectedId(routing, docs.get(0)))); + assertThat(leaf.getId(1), equalTo(expectedId(routing, docs.get(0)))); + assertThat(leaf.getId(2), equalTo(expectedId(routing, docs.get(1)))); }; prepareIndexReader(indexAndForceMerge(routing, docs), verify, false); } From ed51100828a15ef0bbf1042b7bbac8830c9c2d55 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Nov 2023 11:19:28 +0100 Subject: [PATCH 051/125] fix: enable assertion on document id --- .../org/elasticsearch/cluster/routing/IndexRouting.java | 5 +++++ .../index/mapper/TsidExtractingIdFieldMapper.java | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index cd05ca3d523d8..cb0aaca00f444 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.ByteUtils; @@ -271,6 +272,10 @@ public String createId(Map flat, byte[] suffix) { return b.createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); } + public String createId(XContentType sourceType, BytesRef tsid, byte[] suffix) { + return hashSource(sourceType, new BytesArray(tsid)).createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); + } + private static int defaultOnEmpty() { throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index b51608ee9505b..0f9e8633a8a50 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -126,9 +126,9 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra * it always must pass. */ IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); - // assert context.getDynamicMappers().isEmpty() == false - // || context.getDynamicRuntimeFields().isEmpty() == false - // || id.equals(indexRouting.createId(TimeSeriesIdFieldMapper.decodeTsid(tsid), suffix)); + assert context.getDynamicMappers().isEmpty() == false + || context.getDynamicRuntimeFields().isEmpty() == false + || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), tsid, suffix)); assert context.getDynamicMappers().isEmpty() == false || context.getDynamicRuntimeFields().isEmpty() == false || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); From 57a9cdc74e0f5e1bc21092a7db34845ff60b8f7c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Nov 2023 11:27:21 +0100 Subject: [PATCH 052/125] fix: forbidded api --- .../org/elasticsearch/index/mapper/DocumentMapperTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index 3f35852a0f3f0..f5bb5861c30c6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -30,6 +30,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; @@ -355,7 +356,7 @@ public void testTooManyDimensionFields() { }))); assertThat( e.getMessage(), - containsString(String.format("Limit of total %sfields [" + max + "] has been exceeded", dimensionErrorSuffix)) + containsString(String.format(Locale.ROOT, "Limit of total %sfields [" + max + "] has been exceeded", dimensionErrorSuffix)) ); } From 09fa2818e0501d09bef90fab26ceb36e65f71cd8 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Nov 2023 11:57:07 +0100 Subject: [PATCH 053/125] fix: remove assertion After hashing the tsid we cannot decode it anymore. --- .../java/org/elasticsearch/cluster/routing/IndexRouting.java | 5 ----- .../index/mapper/TsidExtractingIdFieldMapper.java | 3 --- 2 files changed, 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index cb0aaca00f444..cd05ca3d523d8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -16,7 +16,6 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.util.ByteUtils; @@ -272,10 +271,6 @@ public String createId(Map flat, byte[] suffix) { return b.createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); } - public String createId(XContentType sourceType, BytesRef tsid, byte[] suffix) { - return hashSource(sourceType, new BytesArray(tsid)).createId(suffix, IndexRouting.ExtractFromSource::defaultOnEmpty); - } - private static int defaultOnEmpty() { throw new IllegalArgumentException("Error extracting routing: source didn't contain any routing fields"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 0f9e8633a8a50..38a626943fc58 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -126,9 +126,6 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra * it always must pass. */ IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); - assert context.getDynamicMappers().isEmpty() == false - || context.getDynamicRuntimeFields().isEmpty() == false - || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), tsid, suffix)); assert context.getDynamicMappers().isEmpty() == false || context.getDynamicRuntimeFields().isEmpty() == false || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); From 1e79c3a343efdd6a33ce58c2d55dcc59ea3cf839 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Nov 2023 12:58:26 +0100 Subject: [PATCH 054/125] fix: do not base64 encode in parseBytesRef --- .../main/java/org/elasticsearch/search/DocValueFormat.java | 2 +- .../java/org/elasticsearch/search/DocValueFormatTests.java | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 8dc218b7db68f..8becbe7a1fe30 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -698,7 +698,7 @@ public Object format(BytesRef value) { @Override public BytesRef parseBytesRef(Object value) { if (value instanceof BytesRef valueAsBytesRef) { - return new BytesRef(Base64.getUrlEncoder().withoutPadding().encodeToString(valueAsBytesRef.bytes)); + return valueAsBytesRef; } if (value instanceof String valueAsString) { return new BytesRef(valueAsString); diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index e2a25635fb335..916637126f29f 100644 --- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -26,6 +26,7 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; @@ -378,8 +379,10 @@ public void testParseTsid() throws IOException { timeSeriesIdBuilder.addLong("long", randomLong()); timeSeriesIdBuilder.addUnsignedLong("ulong", randomLong()); BytesRef expected = timeSeriesIdBuilder.withHash().toBytesRef(); - Object tsidFormat = DocValueFormat.TIME_SERIES_ID.format(expected); - BytesRef actual = DocValueFormat.TIME_SERIES_ID.parseBytesRef(tsidFormat); + BytesRef actual = DocValueFormat.TIME_SERIES_ID.parseBytesRef(expected); assertEquals(expected, actual); + Object tsidFormat = DocValueFormat.TIME_SERIES_ID.format(expected); + Object tsidBase64 = Base64.getUrlEncoder().withoutPadding().encodeToString(expected.bytes); + assertEquals(tsidFormat, tsidBase64); } } From 7632bd96668a5cb389c6d3a27075725e0b66680e Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Nov 2023 13:12:43 +0100 Subject: [PATCH 055/125] fix: update InternalComposite formatting and parsing of tsid --- .../aggregations/bucket/composite/InternalComposite.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java index 3233b6e276f7b..e0c65d0737477 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/InternalComposite.java @@ -13,7 +13,6 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.AggregationReduceContext; import org.elasticsearch.search.aggregations.Aggregations; @@ -522,12 +521,11 @@ static Object formatObject(Object obj, DocValueFormat format) { Object parsed; if (obj.getClass() == BytesRef.class && format == DocValueFormat.TIME_SERIES_ID) { BytesRef value = (BytesRef) obj; + // NOTE: formatting a tsid returns a Base64 encoding of the tsid BytesRef which we cannot use to get back the original tsid formatted = format.format(value); - parsed = format.parseBytesRef(formatted); + parsed = format.parseBytesRef(value); // NOTE: we cannot parse the Base64 encoding representation of the tsid and get back the original BytesRef - String objBase64 = TimeSeriesIdFieldMapper.decodeTsid(value).toString(); - BytesRef objBase64BytesRef = new BytesRef(objBase64); - if (objBase64BytesRef.equals(parsed) == false) { + if (parsed.equals(obj) == false) { throw new IllegalArgumentException( "Format [" + format From 5f17caf5128ee33712f4fdf5bf548e206d3dc31f Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Nov 2023 18:52:36 +0100 Subject: [PATCH 056/125] fix: do not parse the BytesRef --- .../bucket/composite/CompositeAggregationBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java index 396c5c870a364..9cd3ef581c7a1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java @@ -246,10 +246,10 @@ protected AggregatorFactory doBuild( Object obj = after.get(sourceName); if (configs[i].missingBucket() && obj == null) { values[i] = null; - } else if (obj instanceof String + } else if (obj instanceof String s && configs[i].fieldType() != null && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { - values[i] = configs[i].format().parseBytesRef(obj); + values[i] = s; } else if (obj instanceof Comparable c) { values[i] = c; } else if (obj instanceof Map && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { From 64bb5d79501f9ab98a78fe8d46b00086ca431860 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Nov 2023 19:18:56 +0100 Subject: [PATCH 057/125] fix: composite tsid parsing and matching --- .../resources/rest-api-spec/test/tsdb/100_composite.yml | 4 ++-- .../main/java/org/elasticsearch/search/DocValueFormat.java | 2 +- .../bucket/composite/CompositeAggregationBuilder.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml index 88874cc6126d9..3da8d8fa341c7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml @@ -140,13 +140,13 @@ composite aggregation on tsid with after: ] after: { tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o", - date: 1619635860000 + date: 1619635800000 } - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 1 } - - match: { aggregations.tsids.buckets.0.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } + - match: { aggregations.tsids.buckets.0.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.buckets.0.key.date: 1619635860000} - match: { aggregations.tsids.buckets.0.doc_count: 1 } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 8becbe7a1fe30..0ee2d47a6e1d6 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -701,7 +701,7 @@ public BytesRef parseBytesRef(Object value) { return valueAsBytesRef; } if (value instanceof String valueAsString) { - return new BytesRef(valueAsString); + return new BytesRef(Base64.getUrlDecoder().decode(valueAsString)); } return parseBytesRefMap(value); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java index 9cd3ef581c7a1..8d6c12598e31b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java @@ -249,7 +249,7 @@ protected AggregatorFactory doBuild( } else if (obj instanceof String s && configs[i].fieldType() != null && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { - values[i] = s; + values[i] = configs[i].format().parseBytesRef(s); } else if (obj instanceof Comparable c) { values[i] = c; } else if (obj instanceof Map && configs[i].fieldType().getClass() == TimeSeriesIdFieldType.class) { From 697d09c1372dab81421dbf1f863c45aadb75bafb Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Nov 2023 20:51:43 +0100 Subject: [PATCH 058/125] fix: yet another yaml test with tsid --- .../rest-api-spec/test/reindex/100_tsdb.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml index df9d9d903efd6..98996cc9c24be 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml @@ -161,7 +161,7 @@ from tsdb to tsdb: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -228,7 +228,7 @@ from standard with tsdb id to tsdb: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -295,7 +295,7 @@ from standard with random _id to tsdb: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -344,7 +344,7 @@ from tsdb to tsdb modifying timestamp: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: pod}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-05-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-05-28T18:50:23.142Z} @@ -393,7 +393,7 @@ from tsdb to tsdb modifying dimension: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: bubbles}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy5k1S1a8G1YLZL5ZxtVAwhiHw5Ds"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -489,7 +489,7 @@ from tsdb to tsdb created by template while modifying dimension: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: {k8s.pod.uid: 1c4fc7b8-93b7-4ba8-b609-2a48af2f8e39, metricset: bubbles}} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy5k1S1a8G1YLZL5ZxtVAwhiHw5Ds"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} From f96a35ba626bc411ad4a4ef15dee3ab7f60408e1 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 28 Nov 2023 10:20:15 +0100 Subject: [PATCH 059/125] fix: larger byte arrays might result in incorrect Base64 encoding --- .../elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index fd99483b2e8eb..5e2b6472a8863 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -253,8 +253,8 @@ public BytesReference withHash() throws IOException { tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); assert tsidHashIndex == tsidHash.length; - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeBytesRef(new BytesRef(tsidHash, 0, tsidHash.length)); + try (BytesStreamOutput out = new BytesStreamOutput(tsidHash.length)) { + out.write(tsidHash, 0, tsidHash.length); return out.bytes(); } } From 1e7fbc632a42d0c64a7d2453fdf97b0183f522b2 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 28 Nov 2023 13:03:11 +0100 Subject: [PATCH 060/125] fix: a bunch of tsid values in yaml tests --- .../rest-api-spec/test/tsdb/100_composite.yml | 16 ++--- .../test/tsdb/140_routing_path.yml | 17 ++--- .../test/tsdb/25_id_generation.yml | 58 ++++++++--------- .../rest-api-spec/test/tsdb/40_search.yml | 15 +++-- .../test/tsdb/60_add_dimensions.yml | 10 +-- .../test/tsdb/70_dimension_types.yml | 64 +++++++++---------- .../rate/TimeSeriesRateAggregatorTests.java | 12 ++-- .../DownsampleActionSingleNodeTests.java | 1 - .../rest-api-spec/test/analytics/100_tsdb.yml | 6 +- 9 files changed, 99 insertions(+), 100 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml index 3da8d8fa341c7..18c1ebe5130b6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml @@ -92,23 +92,23 @@ composite aggregation on tsid: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } + - match: { aggregations.tsids.buckets.0.key.tsid: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA" } - match: { aggregations.tsids.buckets.0.key.date: 1619635800000} - match: { aggregations.tsids.buckets.0.doc_count: 3 } - - match: { aggregations.tsids.buckets.1.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } + - match: { aggregations.tsids.buckets.1.key.tsid: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA" } - match: { aggregations.tsids.buckets.1.key.date: 1619635860000} - match: { aggregations.tsids.buckets.1.doc_count: 1 } - - match: { aggregations.tsids.buckets.2.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } + - match: { aggregations.tsids.buckets.2.key.tsid: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg" } - match: { aggregations.tsids.buckets.2.key.date: 1619635800000 } - match: { aggregations.tsids.buckets.2.doc_count: 3 } - - match: { aggregations.tsids.buckets.3.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } + - match: { aggregations.tsids.buckets.3.key.tsid: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg" } - match: { aggregations.tsids.buckets.3.key.date: 1619635860000} - match: { aggregations.tsids.buckets.3.doc_count: 1 } - - match: { aggregations.tsids.after_key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } + - match: { aggregations.tsids.after_key.tsid: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg" } - match: { aggregations.tsids.after_key.date: 1619635860000} --- @@ -139,17 +139,17 @@ composite aggregation on tsid with after: } ] after: { - tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o", + tsid: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg", date: 1619635800000 } - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 1 } - - match: { aggregations.tsids.buckets.0.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } + - match: { aggregations.tsids.buckets.0.key.tsid: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg" } - match: { aggregations.tsids.buckets.0.key.date: 1619635860000} - match: { aggregations.tsids.buckets.0.doc_count: 1 } - - match: { aggregations.tsids.after_key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } + - match: { aggregations.tsids.after_key.tsid: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg" } - match: { aggregations.tsids.after_key.date: 1619635860000} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 275bd446d3996..662e6e82a0fd8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -71,21 +71,22 @@ missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key: "JNy0BQX41tKNa3KEdjReXM85ihHDIG1DaFBdVI_fYOQvJgKOvg" } + - match: { aggregations.tsids.buckets.0.key: "M4NGZuPu-7ista-OslulS5oRP-85ihHDoibMoRwXiwRoYqddJop8zQ" } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "JNy0BQX41tKNa3KEdjReXM912oDh9NI69d0Kk5TQ6CAdewYP5A" } + - match: { aggregations.tsids.buckets.1.key: "M4NGZuPu-7ista-OslulS9xRYkR12oDhJ6jyNsEEEXHALSWt58JvoA" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.15, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } + - match: { aggregations.tsids.buckets.2.key: "3LQFBfjW0o1rcoR2NF5czzmKEcMgbUNoUF1Uj99g5C8mAo6-" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } + - match: { aggregations.tsids.buckets.3.key: "3LQFBfjW0o1rcoR2NF5cz3XagOH00jr13QqTlNDoIB17Bg_k" } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 7.30, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 7.15 + , error: 0.01 }} --- missing dimension on routing path field: @@ -198,10 +199,10 @@ multi-value routing path field: - match: {hits.total.value: 4} - length: {aggregations.tsids.buckets: 2} - - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } + - match: {aggregations.tsids.buckets.0.key: "M4NGZuPu-7ista-OslulS5oRP-85ihHDoibMoRwXiwRoYqddJop8zQ" } - match: {aggregations.tsids.buckets.0.doc_count: 2 } - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } + - match: { aggregations.tsids.buckets.1.key: "M4NGZuPu-7ista-OslulS9xRYkR12oDhJ6jyNsEEEXHALSWt58JvoA" } - match: {aggregations.tsids.buckets.1.doc_count: 2 } - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 4e5edc6400e97..6cb3e2a010e39 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -76,7 +76,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} + - match: {items.0.index._id: cZZNsyIXXGJG3XgFAAABeRnS7fM} - do: bulk: @@ -85,7 +85,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} + - match: {items.0.index._id: cZZNsyIXXGJG3XgFAAABeRnS7fM} - do: search: @@ -97,39 +97,39 @@ generates a consistent id: - match: {hits.total.value: 9} - - match: { hits.hits.0._id: cn4excfoxSs_KdA5AAABeRnRFAY } + - match: { hits.hits.0._id: cn4exclok5LVWXitAAABeRnRFAY } - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z } - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.1._id: cZZNs7B9sSWsyrL5AAABeRnRGTM } + - match: { hits.hits.1._id: cZZNsyIXXGJG3XgFAAABeRnRGTM } - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:04.467Z } - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.2._id: cn4excfoxSs_KdA5AAABeRnRYiY } + - match: { hits.hits.2._id: cn4exclok5LVWXitAAABeRnRYiY } - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:23.142Z } - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.3._id: cZZNs7B9sSWsyrL5AAABeRnRZ1M } + - match: { hits.hits.3._id: cZZNsyIXXGJG3XgFAAABeRnRZ1M } - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:24.467Z } - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.4._id: cZZNs7B9sSWsyrL5AAABeRnRtXM } + - match: { hits.hits.4._id: cZZNsyIXXGJG3XgFAAABeRnRtXM } - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:44.467Z } - match: { hits.hits.4._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.5._id: cn4excfoxSs_KdA5AAABeRnR11Y } + - match: { hits.hits.5._id: cn4exclok5LVWXitAAABeRnR11Y } - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:50:53.142Z } - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.6._id: cn4excfoxSs_KdA5AAABeRnR_mY } + - match: { hits.hits.6._id: cn4exclok5LVWXitAAABeRnR_mY } - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:03.142Z } - match: { hits.hits.6._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.7._id: cZZNs7B9sSWsyrL5AAABeRnSA5M } + - match: { hits.hits.7._id: cZZNsyIXXGJG3XgFAAABeRnSA5M } - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:51:04.467Z } - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.8._id: cZZNs7B9sSWsyrL5AAABeRnS7fM } + - match: { hits.hits.8._id: cZZNsyIXXGJG3XgFAAABeRnS7fM } - match: { hits.hits.8._source.@timestamp: 2021-04-28T18:52:04.467Z } - match: { hits.hits.8._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } @@ -170,7 +170,7 @@ index a new document on top of an old one: network: tx: 111434595272 rx: 430605511 - - match: {_id: cn4excfoxSs_KdA5AAABeRnR_mY} + - match: {_id: cn4exclok5LVWXitAAABeRnR_mY} - do: search: @@ -215,7 +215,7 @@ index a new document on top of an old one over bulk: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: {items.0.index._id: cn4excfoxSs_KdA5AAABeRnR_mY} + - match: {items.0.index._id: cn4exclok5LVWXitAAABeRnR_mY} - do: search: @@ -239,7 +239,7 @@ create operation on top of old document fails: reason: id generation changed in 8.2 - do: - catch: "/\\[cn4excfoxSs_KdA5AAABeRnR_mY\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" + catch: "/\\[cn4exclok5LVWXitAAABeRnR_mY\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" index: refresh: true index: test @@ -268,7 +268,7 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[cn4excfoxSs_KdA5AAABeRnR_mY][KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQAAAAAAAAA@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[cn4exclok5LVWXitAAABeRnR_mY][KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: @@ -284,13 +284,13 @@ ids query: - field: k8s.pod.network.tx query: ids: - values: ["cn4excfoxSs_KdA5AAABeRnR_mY", "cn4excfoxSs_KdA5AAABeRnR11Y"] + values: ["cn4exclok5LVWXitAAABeRnR11Y", "cn4exclok5LVWXitAAABeRnR_mY"] sort: ["@timestamp"] - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "cn4excfoxSs_KdA5AAABeRnR11Y"} + - match: {hits.hits.0._id: "cn4exclok5LVWXitAAABeRnR11Y"} - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434587694]} - - match: {hits.hits.1._id: "cn4excfoxSs_KdA5AAABeRnR_mY"} - - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [1434595272]} + - match: {hits.hits.1._id: "cn4exclok5LVWXitAAABeRnR_mY" } + - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [ 1434595272 ]} --- get: @@ -301,9 +301,9 @@ get: - do: get: index: test - id: cZZNs7B9sSWsyrL5AAABeRnSA5M + id: cZZNsyIXXGJG3XgFAAABeRnSA5M - match: {_index: test} - - match: {_id: cZZNs7B9sSWsyrL5AAABeRnSA5M} + - match: {_id: cZZNsyIXXGJG3XgFAAABeRnSA5M} - match: _source: "@timestamp": "2021-04-28T18:51:04.467Z" @@ -351,7 +351,7 @@ delete: - do: delete: index: test - id: cn4excfoxSs_KdA5AAABeRnR_mY + id: cn4exclok5LVWXitAAABeRnR_mY - match: {result: deleted} --- @@ -391,20 +391,20 @@ delete over _bulk: mget: index: test body: - ids: [ cn4excfoxSs_KdA5AAABeRnR_mY, cn4excfoxSs_KdA5AAABeRnR11Y ] + ids: [ cn4exclok5LVWXitAAABeRnR_mY, cn4exclok5LVWXitAAABeRnR11Y ] - match: { docs.0._index: "test" } - - match: { docs.0._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } + - match: { docs.0._id: "cn4exclok5LVWXitAAABeRnR_mY" } - match: { docs.0.found: true } - match: { docs.1._index: "test" } - - match: { docs.1._id: "cn4excfoxSs_KdA5AAABeRnR11Y" } + - match: { docs.1._id: "cn4exclok5LVWXitAAABeRnR11Y" } - match: { docs.1.found: true } - do: bulk: index: test body: - - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR_mY"}}' - - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR11Y"}}' + - '{"delete": {"_id": "cn4exclok5LVWXitAAABeRnR_mY"}}' + - '{"delete": {"_id": "cn4exclok5LVWXitAAABeRnR11Y"}}' - '{"delete": {"_id": "not found ++ not found"}}' - match: {items.0.delete.result: deleted} - match: {items.1.delete.result: deleted} @@ -454,7 +454,7 @@ routing_path matches deep object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"bar": {"baz": {"uid": "uid1"}}}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: OcEOGchJrjH1fFX8AAABeRnRGTM} + - match: {items.0.index._id: OcEOGftorSJSldkIAAABeRnRGTM} --- routing_path matches object: @@ -495,4 +495,4 @@ routing_path matches object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"uid": "uid1"}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: 8bgiqW9JKwAyp1bZAAABeRnRGTM} + - match: {items.0.index._id: 8bgiqa25oA7vLNHsAAABeRnRGTM} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index 3e094cd2eb1ae..7356c08721b23 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -185,7 +185,7 @@ fetch a tag: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} + - match: {hits.hits.0.fields._tsid: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"]} --- aggregate a dimension: @@ -282,9 +282,9 @@ aggregate a tag: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} + - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} + - match: {aggregations.tsids.buckets.1.key: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -319,7 +319,8 @@ sort by tsid: sort: [ "_tsid", "@timestamp" ] - match: {hits.total.value: 8} - - match: {hits.hits.0.sort: ["KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQAAAAAAAAA", 1619635803142]} - - match: {hits.hits.1.sort: ["KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQAAAAAAAAA", 1619635823142]} - - match: {hits.hits.4.sort: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0oAAAAAAAAA", 1619635804467]} - - match: {hits.hits.7.sort: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0oAAAAAAAAA", 1619635864467]} + # NOTE: 98023 - Check possible buffer issue with _tsid Base64 encoding ('AAAAAAAAAAA') + - match: {hits.hits.0.sort: ["KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pAAAAAAAAAAA", 1619635803142]} + - match: {hits.hits.1.sort: ["KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pAAAAAAAAAAA", 1619635823142]} + - match: {hits.hits.4.sort: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSgAAAAAAAAAA", 1619635804467]} + - match: {hits.hits.7.sort: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSgAAAAAAAAAA", 1619635864467]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 4f9089533ba8e..e34d9fccd0253 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -48,7 +48,7 @@ add dimensions with put_mapping: - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ "JNu4XCk2JFwjn2IrkVkU1soGlT_5e6_NYGOZWULpmMG9IAlZlA" ] } + - match: {hits.hits.0.fields._tsid: [ "27hcKTYkXCOfYiuRWRTWygaVP_l7r81gY5lZQumYwb0gCVmU" ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -96,7 +96,7 @@ add dimensions to no dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ "JNu4XCk2JFwjn2IrkVkU1soGlT_5e6_NYGOZWULpmMG9IAlZlA" ] } + - match: {hits.hits.0.fields._tsid: [ "27hcKTYkXCOfYiuRWRTWygaVP_l7r81gY5lZQumYwb0gCVmU" ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -144,7 +144,7 @@ add dimensions to no dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ "JNu4XCk2JFwjn2IrkVkU1soGlT_5e6_NYGOZWULpmMG9IAlZlA" ] } + - match: {hits.hits.0.fields._tsid: [ "27hcKTYkXCOfYiuRWRTWygaVP_l7r81gY5lZQumYwb0gCVmU" ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -196,7 +196,7 @@ add dimensions to some dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: ["KPgl3zOd7TnU3SZ_BkidRdUGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y"] } + - match: {hits.hits.0.fields._tsid: ["-CXfM53tOdTdJn8GSJ1F1QaVP_kGlT_5U9X7I_7uFS_raLrCjO5zpg"] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -247,5 +247,5 @@ add dimensions to some dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: ["KPgl3zOd7TnU3SZ_BkidRdUGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y"] } + - match: {hits.hits.0.fields._tsid: ["-CXfM53tOdTdJn8GSJ1F1QaVP_kGlT_5U9X7I_7uFS_raLrCjO5zpg"] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 74eb8ea9ad4fa..f8000e56b0f1a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -65,10 +65,10 @@ keyword dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "JNy0BQX41tKNa3KEdjReXM85ihHDIG1DaFBdVI_fYOQvJgKOvg"} + - match: {aggregations.tsids.buckets.0.key: "3LQFBfjW0o1rcoR2NF5czzmKEcMgbUNoUF1Uj99g5C8mAo6-"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: "JNy0BQX41tKNa3KEdjReXM912oDh9NI69d0Kk5TQ6CAdewYP5A"} + - match: {aggregations.tsids.buckets.1.key: "3LQFBfjW0o1rcoR2NF5cz3XagOH00jr13QqTlNDoIB17Bg_k"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -144,19 +144,19 @@ flattened dimension: - match: { hits.total.value: 8} - length: { aggregations.tsids.buckets: 4} - - match: { aggregations.tsids.buckets.0.key: "NCLYECP-GoaIfjk0RBfdlg0oaZ29eRDHR3kQx0fjCuTKddqA4R9ytcYdqdbcoJ8VBuvqFtQ" } + - match: { aggregations.tsids.buckets.0.key: "ItgQI_4ahoh-OTREF92WDShpnb15EMdHeRDHR-MK5Mp12oDhH3K1xh2p1tygnxUG6-oW1A" } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "NCLYECP-GoaIfjk0RBfdlg2rnf7qeRDHR3kQx0eLDvTuOYoRw4EeUIKK1KGFPmxqCWQyQJE" } + - match: { aggregations.tsids.buckets.1.key: "ItgQI_4ahoh-OTREF92WDaud_up5EMdHeRDHR4sO9O45ihHDgR5QgorUoYU-bGoJZDJAkQ" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key: "NCLYECP-GoaIfjk0RBfdlg3P1l1UeRDHR3kQx0fjCuTKddqA4ai7CiSuCxA-PhBdLYOXkVY" } + - match: { aggregations.tsids.buckets.2.key: "ItgQI_4ahoh-OTREF92WDc_WXVR5EMdHeRDHR-MK5Mp12oDhqLsKJK4LED4-EF0tg5eRVg" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key: "NCLYECP-GoaIfjk0RBfdlg3gKelzeRDHR3kQx0eLDvTuOYoRw6Wapg_P07YIhdV3NFJDjjE" } + - match: { aggregations.tsids.buckets.3.key: "ItgQI_4ahoh-OTREF92WDeAp6XN5EMdHeRDHR4sO9O45ihHDpZqmD8_TtgiF1Xc0UkOOMQ" } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.65, error: 0.01 }} @@ -232,11 +232,11 @@ flattened empty dimension: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 2 } - - match: { aggregations.tsids.buckets.0.key: "JNy0BQX41tKNa3KEdjReXM85ihHDIG1DaFBdVI_fYOQvJgKOvg" } + - match: { aggregations.tsids.buckets.0.key: "3LQFBfjW0o1rcoR2NF5czzmKEcMgbUNoUF1Uj99g5C8mAo6-" } - match: { aggregations.tsids.buckets.0.doc_count: 4 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "JNy0BQX41tKNa3KEdjReXM912oDh9NI69d0Kk5TQ6CAdewYP5A" } + - match: { aggregations.tsids.buckets.1.key: "3LQFBfjW0o1rcoR2NF5cz3XagOH00jr13QqTlNDoIB17Bg_k" } - match: { aggregations.tsids.buckets.1.doc_count: 4 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.22, error: 0.01 }} @@ -312,29 +312,29 @@ flattened field missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 6 } - - match: { aggregations.tsids.buckets.0.key.: "LPBzSKpPysYDR-l1jvYA8jPxTXebeRDHRzmKEcOUW9u_kzgC7pSzoi1utVcm" } + - match: { aggregations.tsids.buckets.0.key.: "XXEWRB81VefgRtsBbneB7_FNd5soaZ29eRDHR3XagOHXoW1-8_85uSUJruBcNyZ0" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "LPBzSKpPysYDR-l1jvYA8jPxTXebeRDHR3XagOEjGQJfwIB2Q5kLDL76leH4" } - - match: { aggregations.tsids.buckets.1.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} + - match: { aggregations.tsids.buckets.1.key: "XXEWRB81VefgRtsBbneB7_FNd5urnf7qeRDHRzmKEcNTc-7NMjWYz3yBCqqKEOhk" } + - match: { aggregations.tsids.buckets.1.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key: "MF1xFkQfNVXn4EbbAW53ge_xTXebKGmdvXkQx0d12oDh16FtfvP_ObklCa7gXDcmdA" } - - match: { aggregations.tsids.buckets.2.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.40, error: 0.01 }} + - match: { aggregations.tsids.buckets.2.key: "XXEWRB81VefgRtsBbneB7_FNd5vP1l1UeRDHR3XagOGLVJxylWdECGmMKtaGoWN9" } + - match: { aggregations.tsids.buckets.2.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key: "MF1xFkQfNVXn4EbbAW53ge_xTXebq53-6nkQx0c5ihHDU3PuzTI1mM98gQqqihDoZA" } - - match: { aggregations.tsids.buckets.3.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} + - match: { aggregations.tsids.buckets.3.key: "XXEWRB81VefgRtsBbneB7_FNd5vgKelzeRDHRzmKEcOppLmEeAsCbQTdeQcbt92R" } + - match: { aggregations.tsids.buckets.3.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} - - match: { aggregations.tsids.buckets.4.key: "MF1xFkQfNVXn4EbbAW53ge_xTXebz9ZdVHkQx0d12oDhi1SccpVnRAhpjCrWhqFjfQ" } - - match: { aggregations.tsids.buckets.4.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 7.10, error: 0.01 }} + - match: { aggregations.tsids.buckets.4.key: "8HNIqk_KxgNH6XWO9gDyM_FNd5t5EMdHOYoRw5Rb27-TOALulLOiLW61VyY" } + - match: { aggregations.tsids.buckets.4.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.5.key: "MF1xFkQfNVXn4EbbAW53ge_xTXeb4Cnpc3kQx0c5ihHDqaS5hHgLAm0E3XkHG7fdkQ" } + - match: { aggregations.tsids.buckets.5.key: "8HNIqk_KxgNH6XWO9gDyM_FNd5t5EMdHddqA4SMZAl_AgHZDmQsMvvqV4fg" } - match: { aggregations.tsids.buckets.5.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.5.voltage.value: { value: 6.60, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.5.voltage.value: { value: 7.30, error: 0.01 }} --- flattened field misspelled routing path field: @@ -412,19 +412,19 @@ flattened field misspelled routing path field: - match: { hits.total.value: 6 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key: "JBs0-JZ2yoAg-Lrw35Mu3ysoaZ29egRdyNeHXPSPghDVzguaRg" } + - match: { aggregations.tsids.buckets.0.key: "GzT4lnbKgCD4uvDfky7fKyhpnb16BF3I14dc9I-CENXOC5pG" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "JBs0-JZ2yoAg-Lrw35Mu3yurnf7qs9-VXFZ6jjZCbl_iiXSs7Q" } + - match: { aggregations.tsids.buckets.1.key: "GzT4lnbKgCD4uvDfky7fK6ud_uqz35VcVnqONkJuX-KJdKzt" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key: "JBs0-JZ2yoAg-Lrw35Mu3yvP1l1UlmlXEQVNXrHpUvpn7by0jA" } + - match: { aggregations.tsids.buckets.2.key: "GzT4lnbKgCD4uvDfky7fK8_WXVSWaVcRBU1eselS-mftvLSM" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key: "JBs0-JZ2yoAg-Lrw35Mu3yvgKelz9WSJqzeYh7aza_7yxDXMZA" } + - match: { aggregations.tsids.buckets.3.key: "GzT4lnbKgCD4uvDfky7fK-Ap6XP1ZImrN5iHtrNr_vLENcxk" } - match: { aggregations.tsids.buckets.3.doc_count: 1 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} @@ -496,10 +496,10 @@ long dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KMaueSdBhc_WIhY4xoPE2EdDgKYd73outpXn7LJV-gQfvlrec7NyMho"} + - match: {aggregations.tsids.buckets.0.key: "xq55J0GFz9YiFjjGg8TYR0OAph3vei62lefsslX6BB--Wt5zs3IyGg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: "KMaueSdBhc_WIhY4xoPE2Eetm41v73outmoSTcFmfQBYjOjMaOWM5zs"} + - match: {aggregations.tsids.buckets.1.key: "xq55J0GFz9YiFjjGg8TYR62bjW_vei62ahJNwWZ9AFiM6Mxo5YznOw"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -571,10 +571,10 @@ ip dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KLVrd7E8oZLyd-tfSm0H6jDYzb-v73outosB-eDb_nzTrIVsJFVQR2c"} + - match: {aggregations.tsids.buckets.0.key: "tWt3sTyhkvJ3619KbQfqMNjNv6_vei62iwH54Nv-fNOshWwkVVBHZw"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: "KLVrd7E8oZLyd-tfSm0H6jDaaOFK73outkqpG8R65Gm4VUhpyuc11zw"} + - match: {aggregations.tsids.buckets.1.key: "tWt3sTyhkvJ3619KbQfqMNpo4Urvei62SqkbxHrkabhVSGnK5zXXPA"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index 4b7688209b81f..42dac7595f3c5 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -55,13 +55,11 @@ public void testSimple() throws IOException { Consumer verifier = r -> { assertThat(r.getBuckets(), hasSize(2)); assertThat( - ((Rate) r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2LDvTu5MAoxgghmvEUd5a_i3Y3TA").getAggregations().asList().get(0)) - .getValue(), + ((Rate) r.getBucketByKey("UVTLXgL1RxfcycWewWul_YsO9O7kwCjGCCGa8RR3lr-LdjdM").getAggregations().asList().get(0)).getValue(), closeTo(59.0 / 3000.0 * MILLIS_IN_SECOND, 0.00001) ); assertThat( - ((Rate) r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2hRiY7lpxc5PgA4k16syGbZGgr0g").getAggregations().asList().get(0)) - .getValue(), + ((Rate) r.getBucketByKey("UVTLXgL1RxfcycWewWul_aFGJjuWnFzk-ADiTXqzIZtkaCvS").getAggregations().asList().get(0)).getValue(), closeTo(206.0 / 4000.0 * MILLIS_IN_SECOND, 0.00001) ); }; @@ -85,10 +83,10 @@ public void testNestedWithinDateHistogram() throws IOException { Consumer verifier = r -> { assertThat(r.getBuckets(), hasSize(2)); assertThat( - r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2LDvTu5MAoxgghmvEUd5a_i3Y3TA"), + r.getBucketByKey("UVTLXgL1RxfcycWewWul_YsO9O7kwCjGCCGa8RR3lr-LdjdM"), instanceOf(InternalTimeSeries.InternalBucket.class) ); - InternalDateHistogram hb = r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2LDvTu5MAoxgghmvEUd5a_i3Y3TA").getAggregations().get("date"); + InternalDateHistogram hb = r.getBucketByKey("UVTLXgL1RxfcycWewWul_YsO9O7kwCjGCCGa8RR3lr-LdjdM").getAggregations().get("date"); { Rate rate = hb.getBuckets().get(1).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((60 - 37 + 14) / 2000.0 * MILLIS_IN_SECOND, 0.00001)); @@ -97,7 +95,7 @@ public void testNestedWithinDateHistogram() throws IOException { Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((37 - 15) / 1000.0 * MILLIS_IN_SECOND, 0.00001)); } - hb = r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2hRiY7lpxc5PgA4k16syGbZGgr0g").getAggregations().get("date"); + hb = r.getBucketByKey("UVTLXgL1RxfcycWewWul_aFGJjuWnFzk-ADiTXqzIZtkaCvS").getAggregations().get("date"); { Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((150 - 74) / 1000.0 * MILLIS_IN_SECOND, 0.00001)); diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index 8beca42bf2a5f..c91f26243de5a 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -933,7 +933,6 @@ public void testResumeDownsamplePartial() throws IOException { // NOTE: there is just one dimension with two possible values, this needs to be one of the two possible tsid values. new BytesRef( new byte[] { - 0x24, 0x42, (byte) 0xe4, (byte) 0x9f, diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml index 2c99a691f127b..14359198a8493 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml @@ -78,11 +78,11 @@ aggretate multi_terms: - field: k8s.pod.ip - length: { aggregations.m_terms.buckets: 3 } - - match: { aggregations.m_terms.buckets.0.key_as_string: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ|10.10.55.3" } + - match: { aggregations.m_terms.buckets.0.key_as_string: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA|10.10.55.3" } - match: { aggregations.m_terms.buckets.0.doc_count: 4 } - - match: { aggregations.m_terms.buckets.1.key_as_string: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o|10.10.55.1" } + - match: { aggregations.m_terms.buckets.1.key_as_string: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg|10.10.55.1" } - match: { aggregations.m_terms.buckets.1.doc_count: 3 } - - match: { aggregations.m_terms.buckets.2.key_as_string: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o|10.10.55.2" } + - match: { aggregations.m_terms.buckets.2.key_as_string: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg|10.10.55.2" } - match: { aggregations.m_terms.buckets.2.doc_count: 1 } --- From dc0dd9009bd9e34ef5c3e1cc4d788a83164e4556 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 28 Nov 2023 15:02:40 +0100 Subject: [PATCH 061/125] fix: some more tsid values --- .../rest-api-spec/test/70_time_series.yml | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml index 5d4a971c6fd1b..426342f92cf14 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml +++ b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml @@ -64,10 +64,10 @@ fetch the _tsid: sort: [ _tsid ] - match: {hits.total.value: 2} - - match: {hits.hits.0.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} + - match: {hits.hits.0.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrZxxTOVygnDHou1_fh12DmF43JAgA"]} - match: {hits.hits.0.fields.metricset: [aa]} - match: {hits.hits.0.fields.ul: [9223372036854775807]} - - match: {hits.hits.1.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} + - match: {hits.hits.1.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q"]} - match: {hits.hits.1.fields.metricset: [aa]} - match: {hits.hits.1.fields.ul: [9223372036854775808]} @@ -119,13 +119,13 @@ aggregate the _tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62VWzfUKafNxTRdD8v6_Z8PbKJ6TQ"} + - match: {aggregations.tsids.buckets.0.key: "SA-a3wlMp7tYDqIHwUtgpu96LrZVbN9Qpp83FNF0Py_r9nw9sonpNA"} - match: {aggregations.tsids.buckets.0.doc_count: 1} - - match: {aggregations.tsids.buckets.1.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"} + - match: {aggregations.tsids.buckets.1.key: "SA-a3wlMp7tYDqIHwUtgpu96LrZxxTOVygnDHou1_fh12DmF43JAgA"} - match: {aggregations.tsids.buckets.1.doc_count: 3} - - match: {aggregations.tsids.buckets.2.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62erJQlP1_gz9tjb7S743_9tpXyfM"} + - match: {aggregations.tsids.buckets.2.key: "SA-a3wlMp7tYDqIHwUtgpu96LrZ6slCU_X-DP22NvtLvjf_22lfJ8w"} - match: {aggregations.tsids.buckets.2.doc_count: 1} - - match: {aggregations.tsids.buckets.3.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"} + - match: {aggregations.tsids.buckets.3.key: "SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q"} - match: {aggregations.tsids.buckets.3.doc_count: 3} @@ -147,14 +147,14 @@ sort by tsid: - match: {hits.total.value: 8 } - - match: {hits.hits.0.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62VWzfUKafNxTRdD8v6_Z8PbKJ6TQ"]} - - match: {hits.hits.1.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} - - match: {hits.hits.2.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} - - match: {hits.hits.3.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} - - match: {hits.hits.4.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62erJQlP1_gz9tjb7S743_9tpXyfM"]} - - match: {hits.hits.5.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} - - match: {hits.hits.6.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} - - match: {hits.hits.7.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} + - match: {hits.hits.0.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrZVbN9Qpp83FNF0Py_r9nw9sonpNA"]} + - match: {hits.hits.1.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrZxxTOVygnDHou1_fh12DmF43JAgA"]} + - match: {hits.hits.2.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrZxxTOVygnDHou1_fh12DmF43JAgA"]} + - match: {hits.hits.3.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrZxxTOVygnDHou1_fh12DmF43JAgA"]} + - match: {hits.hits.4.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrZ6slCU_X-DP22NvtLvjf_22lfJ8w"]} + - match: {hits.hits.5.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q"]} + - match: {hits.hits.6.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q"]} + - match: {hits.hits.7.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q"]} --- composite aggregation on tsid: @@ -180,13 +180,13 @@ composite aggregation on tsid: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62VWzfUKafNxTRdD8v6_Z8PbKJ6TQ" } + - match: { aggregations.tsids.buckets.0.key.tsid: "SA-a3wlMp7tYDqIHwUtgpu96LrZVbN9Qpp83FNF0Py_r9nw9sonpNA" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - - match: { aggregations.tsids.buckets.1.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA" } + - match: { aggregations.tsids.buckets.1.key.tsid: "SA-a3wlMp7tYDqIHwUtgpu96LrZxxTOVygnDHou1_fh12DmF43JAgA" } - match: { aggregations.tsids.buckets.1.doc_count: 3 } - - match: { aggregations.tsids.buckets.2.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62erJQlP1_gz9tjb7S743_9tpXyfM" } + - match: { aggregations.tsids.buckets.2.key.tsid: "SA-a3wlMp7tYDqIHwUtgpu96LrZ6slCU_X-DP22NvtLvjf_22lfJ8w" } - match: { aggregations.tsids.buckets.2.doc_count: 1 } - - match: { aggregations.tsids.buckets.3.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U" } + - match: { aggregations.tsids.buckets.3.key.tsid: "SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q" } - match: { aggregations.tsids.buckets.3.doc_count: 3 } - - match: { aggregations.tsids.after_key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U" } + - match: { aggregations.tsids.after_key.tsid: "SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q" } From bf94a0f04031cbbd7b5adc1c34eb947f06893de5 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 28 Nov 2023 15:42:39 +0100 Subject: [PATCH 062/125] fix: some more tsid values --- .../metrics/geoline-aggregation.asciidoc | 20 +++---- .../timeseries/TimeSeriesAggregatorTests.java | 30 +++++----- .../TimeSeriesAggregatorTsidHashingTests.java | 30 +++++----- .../test/data_stream/150_tsdb.yml | 6 +- .../rest-api-spec/test/reindex/100_tsdb.yml | 8 +-- .../rest-api-spec/test/delete/70_tsdb.yml | 8 +-- .../rest-api-spec/test/tsdb/30_snapshot.yml | 2 +- .../rest-api-spec/test/tsdb/50_alias.yml | 10 ++-- .../test/tsdb/80_index_resize.yml | 4 +- .../test/analytics/reset_tracking_rate.yml | 2 +- .../test/security/authz/70_tsdb.yml | 4 +- .../test/spatial/120_position_geo_line.yml | 60 +++++++++---------- 12 files changed, 92 insertions(+), 92 deletions(-) diff --git a/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc b/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc index 98b3593ab1ba9..cef25fdf401f0 100644 --- a/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc @@ -347,8 +347,8 @@ This query will result in: "aggregations": { "path": { "buckets": { - "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": { - "key": "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA", + "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0": { + "key": "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0", "doc_count": 2, "museum_tour": { "type": "Feature", @@ -361,13 +361,13 @@ This query will result in: } } }, - "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": { - "key": "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw", - "doc_count": 3, + "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A": { + "key": "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A", + "doc_count": 5, "museum_tour": { "type": "Feature", "geometry": { - "coordinates": [ [ 4.401384, 51.220292 ], [ 4.405819, 51.221758 ], [ 4.4052, 51.2229 ] ], + "coordinates": [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ], "type": "LineString" }, "properties": { @@ -375,13 +375,13 @@ This query will result in: } } }, - "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": { - "key": "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA", - "doc_count": 5, + "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH": { + "key": "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH", + "doc_count": 3, "museum_tour": { "type": "Feature", "geometry": { - "coordinates": [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ], + "coordinates": [ [ 4.401384, 51.220292 ], [ 4.405819, 51.221758 ], [ 4.4052, 51.2229 ] ], "type": "LineString" }, "properties": { diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index 7767ec9595950..d2581118bbe69 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -66,19 +66,19 @@ public void testStandAloneTimeSeriesWithSum() throws IOException { }, ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); + assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").docCount, equalTo(4L)); assertThat( - ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").getAggregations().get("sum")).value(), + ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").getAggregations().get("sum")).value(), equalTo(22.0) ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); + assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").docCount, equalTo(2L)); assertThat( - ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").getAggregations().get("sum")).value(), + ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").getAggregations().get("sum")).value(), equalTo(6.0) ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); + assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").docCount, equalTo(2L)); assertThat( - ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").getAggregations().get("sum")).value(), + ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").getAggregations().get("sum")).value(), equalTo(8.0) ); @@ -161,24 +161,24 @@ public void testMultiBucketAggregationAsSubAggregation() throws IOException { Consumer verifier = ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); - InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o") + assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").docCount, equalTo(4L)); + InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg") .getAggregations() .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 4, false, null, InternalAggregations.EMPTY)) ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); - byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg") + assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").docCount, equalTo(2L)); + byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA") .getAggregations() .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); - byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4") + assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").docCount, equalTo(2L)); + byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg") .getAggregations() .get("by_timestamp"); assertThat( @@ -200,9 +200,9 @@ public void testAggregationSize() throws IOException { List> verifiers = new ArrayList>(); - verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").docCount, equalTo(4L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").docCount, equalTo(2L))); for (int i = 1; i <= 3; i++) { int size = i; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java index f49ac048a1181..39785252f8708 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java @@ -95,19 +95,19 @@ public void testStandAloneTimeSeriesWithSum() throws IOException { }, ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); + assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").docCount, equalTo(2L)); assertThat( - ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").getAggregations().get("sum")).value(), + ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").getAggregations().get("sum")).value(), equalTo(6.0) ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); + assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").docCount, equalTo(2L)); assertThat( - ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").getAggregations().get("sum")).value(), + ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").getAggregations().get("sum")).value(), equalTo(8.0) ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); + assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").docCount, equalTo(4L)); assertThat( - ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").getAggregations().get("sum")).value(), + ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").getAggregations().get("sum")).value(), equalTo(22.0) ); @@ -190,24 +190,24 @@ public void testMultiBucketAggregationAsSubAggregation() throws IOException { Consumer verifier = ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); - InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg") + assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").docCount, equalTo(2L)); + InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA") .getAggregations() .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); - byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4") + assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").docCount, equalTo(2L)); + byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg") .getAggregations() .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); - byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o") + assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").docCount, equalTo(4L)); + byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg") .getAggregations() .get("by_timestamp"); assertThat( @@ -229,9 +229,9 @@ public void testAggregationSize() throws IOException { List> verifiers = new ArrayList>(); - verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").docCount, equalTo(4L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").docCount, equalTo(2L))); for (int i = 1; i <= 3; i++) { int size = i; diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 442ba800200a2..d35ec80299954 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -147,7 +147,7 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" ]} + - match: {hits.hits.0.fields._tsid: [ "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg" ]} --- "aggregate the tsid": @@ -168,9 +168,9 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} + - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} + - match: {aggregations.tsids.buckets.1.key: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml index 98996cc9c24be..0b2a8a590e6c6 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml @@ -161,7 +161,7 @@ from tsdb to tsdb: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} + - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lL6UpbL76qjIUtiX4BxwB2rktJiZ2LipiA"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -228,7 +228,7 @@ from standard with tsdb id to tsdb: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} + - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lL6UpbL76qjIUtiX4BxwB2rktJiZ2LipiA"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -295,7 +295,7 @@ from standard with random _id to tsdb: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} + - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lL6UpbL76qjIUtiX4BxwB2rktJiZ2LipiA"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -344,7 +344,7 @@ from tsdb to tsdb modifying timestamp: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} + - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lL6UpbL76qjIUtiX4BxwB2rktJiZ2LipiA"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-05-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-05-28T18:50:23.142Z} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml index ce61e0a0711d0..2fe56ee00614c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml @@ -43,15 +43,15 @@ location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } + - match: { _id: crxuhKULvxRQ9efYAAABiHD35_g } - match: { result: created } - match: { _version: 1 } - do: delete: index: weather_sensors - id: crxuhAep5Npwt_etAAABiHD35_g - - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } + id: crxuhKULvxRQ9efYAAABiHD35_g + - match: { _id: crxuhKULvxRQ9efYAAABiHD35_g } - match: { result: deleted } - match: { _version: 2 } @@ -68,6 +68,6 @@ location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } + - match: { _id: crxuhKULvxRQ9efYAAABiHD35_g } - match: { result: created } - match: { _version: 3 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 142e17e4f665c..6424c74d93413 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -148,5 +148,5 @@ teardown: - match: {hits.total.value: 1} - match: {hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507} - - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} + - match: {hits.hits.0.fields._tsid: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index c1b304cb1c155..fe83794064848 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -85,9 +85,9 @@ search an alias: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} + - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} + - match: {aggregations.tsids.buckets.1.key: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -129,10 +129,10 @@ index into alias: _key: asc - match: {hits.total.value: 12} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} + - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} + - match: {aggregations.tsids.buckets.1.key: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - match: {aggregations.tsids.buckets.2.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} + - match: {aggregations.tsids.buckets.2.key: "KMQn1H8GA7xNFfZA53p2lL6UpbL76qjIUtiX4BxwB2rktJiZ2LipiA"} - match: {aggregations.tsids.buckets.2.doc_count: 4} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index 4e3bc2510e55c..c710a5bca8d0a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -124,7 +124,7 @@ shrink: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} + - match: {hits.hits.0.fields._tsid: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"]} --- clone: @@ -148,4 +148,4 @@ clone: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} + - match: {hits.hits.0.fields._tsid: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"]} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml index e6e9a8a0aa295..37a5444c6d235 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml @@ -129,7 +129,7 @@ setup: - match: { hits.total.value: 9 } - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: "JEMc3XgEDWzF216sBF5nxwW5AZB2-M4K_VYiHrKDjUNX3zHVkw" } + - match: { aggregations.ts.buckets.0.key: "QxzdeAQNbMXbXqwEXmfHBbkBkHb4zgr9ViIesoONQ1ffMdWT" } - match: { aggregations.ts.buckets.0.doc_count: 9 } - gte: { aggregations.ts.buckets.0.rate.value: 0.0 } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml index 66ce482f70603..b463a2cbaf4f7 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml @@ -104,7 +104,7 @@ document level security on tag: - match: {hits.total.value: 4} - length: {aggregations.tsids.buckets: 1} - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} + - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} --- @@ -151,7 +151,7 @@ document level security on dimension: - match: { hits.total.value: 4 } - length: { aggregations.tsids.buckets: 1 } - - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} + - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} --- diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml index 85e1e1a5800ad..2df4187b0032a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml @@ -87,8 +87,8 @@ setup: - length: { aggregations.by_time_series.buckets: 3 } - match: aggregations.by_time_series.buckets: - "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": - key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" + "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0": + key: "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0" doc_count: 2 museum_tour: type: Feature @@ -96,8 +96,8 @@ setup: coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] type: LineString properties: { complete: true } - "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": - key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" + "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A": + key: "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A" doc_count: 5 museum_tour: type: Feature @@ -105,8 +105,8 @@ setup: coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] type: LineString properties: { complete: true } - "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": - key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" + "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH": + key: "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH" doc_count: 3 museum_tour: type: Feature @@ -140,8 +140,8 @@ setup: - length: { aggregations.by_time_series.buckets: 3 } - match: aggregations.by_time_series.buckets: - "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": - key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" + "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0": + key: "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0" doc_count: 2 museum_tour: type: Feature @@ -149,8 +149,8 @@ setup: coordinates: [ [ 2.327, 48.86 ], [ 2.336389, 48.861111 ] ] type: LineString properties: { complete: true } - "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": - key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" + "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A": + key: "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A" doc_count: 5 museum_tour: type: Feature @@ -162,8 +162,8 @@ setup: [ 4.889187, 52.373184 ] ] type: LineString properties: { complete: true } - "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": - key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" + "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH": + key: "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH" doc_count: 3 museum_tour: type: Feature @@ -216,8 +216,8 @@ setup: - length: { aggregations.by_time_series.buckets: 3 } - match: aggregations.by_time_series.buckets: - "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": - key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" + "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0": + key: "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0" doc_count: 2 museum_tour: type: Feature @@ -225,8 +225,8 @@ setup: coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] type: LineString properties: { complete: true } - "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": - key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" + "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A": + key: "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A" doc_count: 5 museum_tour: type: Feature @@ -234,8 +234,8 @@ setup: coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] type: LineString properties: { complete: true } - "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": - key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" + "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH": + key: "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH" doc_count: 3 museum_tour: type: Feature @@ -326,8 +326,8 @@ setup: - length: { aggregations.with_time_series.buckets: 3 } - match: aggregations.with_time_series.buckets: - "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": - key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" + "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0": + key: "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0" doc_count: 2 museums: doc_count_error_upper_bound: 0 @@ -342,8 +342,8 @@ setup: type: LineString properties: complete: true - "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": - key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" + "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A": + key: "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A" doc_count: 5 museums: doc_count_error_upper_bound: 0 @@ -358,8 +358,8 @@ setup: type: LineString properties: complete: true - "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": - key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" + "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH": + key: "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH" doc_count: 3 museums: doc_count_error_upper_bound: 0 @@ -447,8 +447,8 @@ setup: - length: { aggregations.with_time_series.buckets: 3 } - match: aggregations.with_time_series.buckets: - "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": - key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" + "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0": + key: "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0" doc_count: 2 museums: doc_count_error_upper_bound: 0 @@ -463,8 +463,8 @@ setup: type: LineString properties: complete: true - "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": - key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" + "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A": + key: "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A" doc_count: 5 museums: doc_count_error_upper_bound: 0 @@ -488,8 +488,8 @@ setup: type: LineString properties: complete: true - "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": - key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" + "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH": + key: "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH" doc_count: 3 museums: doc_count_error_upper_bound: 0 From 28b07325eca6d7c6b87181ca8a5b75aef496b99c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 28 Nov 2023 16:45:43 +0100 Subject: [PATCH 063/125] fix: some tsid values and modified yaml test with bug --- .../rest-api-spec/test/tsdb/40_search.yml | 18 ++++++--- .../rest-api-spec/test/70_time_series.yml | 38 +++++++++---------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index 7356c08721b23..1275411194e23 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -316,11 +316,19 @@ sort by tsid: search: index: test body: + fields: [ "_tsid" ] sort: [ "_tsid", "@timestamp" ] - match: {hits.total.value: 8} - # NOTE: 98023 - Check possible buffer issue with _tsid Base64 encoding ('AAAAAAAAAAA') - - match: {hits.hits.0.sort: ["KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pAAAAAAAAAAA", 1619635803142]} - - match: {hits.hits.1.sort: ["KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pAAAAAAAAAAA", 1619635823142]} - - match: {hits.hits.4.sort: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSgAAAAAAAAAA", 1619635804467]} - - match: {hits.hits.7.sort: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSgAAAAAAAAAA", 1619635864467]} + # NOTE: 98023 - Bug: the value returned by 'sort' and 'fields' API should be the same + - match: {hits.hits.0.sort: ["KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA", 1619635803142]} + - match: {hits.hits.0.fields._tsid: [ "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA"]} + + - match: {hits.hits.1.sort: ["KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA", 1619635823142]} + - match: {hits.hits.1.fields._tsid: [ "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA"]} + + - match: {hits.hits.4.sort: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg", 1619635804467]} + - match: {hits.hits.4.fields._tsid: [ "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"]} + + - match: {hits.hits.7.sort: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg", 1619635864467]} + - match: {hits.hits.7.fields._tsid: [ "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"]} diff --git a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml index 426342f92cf14..5d4a971c6fd1b 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml +++ b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/70_time_series.yml @@ -64,10 +64,10 @@ fetch the _tsid: sort: [ _tsid ] - match: {hits.total.value: 2} - - match: {hits.hits.0.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrZxxTOVygnDHou1_fh12DmF43JAgA"]} + - match: {hits.hits.0.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} - match: {hits.hits.0.fields.metricset: [aa]} - match: {hits.hits.0.fields.ul: [9223372036854775807]} - - match: {hits.hits.1.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q"]} + - match: {hits.hits.1.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} - match: {hits.hits.1.fields.metricset: [aa]} - match: {hits.hits.1.fields.ul: [9223372036854775808]} @@ -119,13 +119,13 @@ aggregate the _tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "SA-a3wlMp7tYDqIHwUtgpu96LrZVbN9Qpp83FNF0Py_r9nw9sonpNA"} + - match: {aggregations.tsids.buckets.0.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62VWzfUKafNxTRdD8v6_Z8PbKJ6TQ"} - match: {aggregations.tsids.buckets.0.doc_count: 1} - - match: {aggregations.tsids.buckets.1.key: "SA-a3wlMp7tYDqIHwUtgpu96LrZxxTOVygnDHou1_fh12DmF43JAgA"} + - match: {aggregations.tsids.buckets.1.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"} - match: {aggregations.tsids.buckets.1.doc_count: 3} - - match: {aggregations.tsids.buckets.2.key: "SA-a3wlMp7tYDqIHwUtgpu96LrZ6slCU_X-DP22NvtLvjf_22lfJ8w"} + - match: {aggregations.tsids.buckets.2.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62erJQlP1_gz9tjb7S743_9tpXyfM"} - match: {aggregations.tsids.buckets.2.doc_count: 1} - - match: {aggregations.tsids.buckets.3.key: "SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q"} + - match: {aggregations.tsids.buckets.3.key: "KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"} - match: {aggregations.tsids.buckets.3.doc_count: 3} @@ -147,14 +147,14 @@ sort by tsid: - match: {hits.total.value: 8 } - - match: {hits.hits.0.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrZVbN9Qpp83FNF0Py_r9nw9sonpNA"]} - - match: {hits.hits.1.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrZxxTOVygnDHou1_fh12DmF43JAgA"]} - - match: {hits.hits.2.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrZxxTOVygnDHou1_fh12DmF43JAgA"]} - - match: {hits.hits.3.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrZxxTOVygnDHou1_fh12DmF43JAgA"]} - - match: {hits.hits.4.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrZ6slCU_X-DP22NvtLvjf_22lfJ8w"]} - - match: {hits.hits.5.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q"]} - - match: {hits.hits.6.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q"]} - - match: {hits.hits.7.fields._tsid: ["SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q"]} + - match: {hits.hits.0.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62VWzfUKafNxTRdD8v6_Z8PbKJ6TQ"]} + - match: {hits.hits.1.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} + - match: {hits.hits.2.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} + - match: {hits.hits.3.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA"]} + - match: {hits.hits.4.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62erJQlP1_gz9tjb7S743_9tpXyfM"]} + - match: {hits.hits.5.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} + - match: {hits.hits.6.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} + - match: {hits.hits.7.fields._tsid: ["KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U"]} --- composite aggregation on tsid: @@ -180,13 +180,13 @@ composite aggregation on tsid: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key.tsid: "SA-a3wlMp7tYDqIHwUtgpu96LrZVbN9Qpp83FNF0Py_r9nw9sonpNA" } + - match: { aggregations.tsids.buckets.0.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62VWzfUKafNxTRdD8v6_Z8PbKJ6TQ" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - - match: { aggregations.tsids.buckets.1.key.tsid: "SA-a3wlMp7tYDqIHwUtgpu96LrZxxTOVygnDHou1_fh12DmF43JAgA" } + - match: { aggregations.tsids.buckets.1.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62ccUzlcoJwx6Ltf34ddg5heNyQIA" } - match: { aggregations.tsids.buckets.1.doc_count: 3 } - - match: { aggregations.tsids.buckets.2.key.tsid: "SA-a3wlMp7tYDqIHwUtgpu96LrZ6slCU_X-DP22NvtLvjf_22lfJ8w" } + - match: { aggregations.tsids.buckets.2.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62erJQlP1_gz9tjb7S743_9tpXyfM" } - match: { aggregations.tsids.buckets.2.doc_count: 1 } - - match: { aggregations.tsids.buckets.3.key.tsid: "SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q" } + - match: { aggregations.tsids.buckets.3.key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U" } - match: { aggregations.tsids.buckets.3.doc_count: 3 } - - match: { aggregations.tsids.after_key.tsid: "SA-a3wlMp7tYDqIHwUtgpu96LrbFuQymKeT4jZyRJafH69cXrsdD5Q" } + - match: { aggregations.tsids.after_key.tsid: "KEgPmt8JTKe7WA6iB8FLYKbvei62xbkMpink-I2ckSWnx-vXF67HQ-U" } From b843b66aa9a2c748da8ab3113894d6d6f4b7317c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 28 Nov 2023 17:42:52 +0100 Subject: [PATCH 064/125] fix: TsidExtractingIdFieldMapperTests --- .../TsidExtractingIdFieldMapperTests.java | 140 +++++++++--------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index 199ff2e52f803..d0b63141f7edd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -82,8 +82,8 @@ public static Iterable params() { items.add( new TestCase( "2022-01-01T01:00:00Z", - "XsFI2ajcFfi45iV3AAABfhMmioA", - "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "XsFI2S5yHCiDlfM-AAABfhMmioA", + "lIs2K8LG_eENNC1Z3qoZTAaVP_l7r81gY5lZQumYwb0gCVmU", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -94,8 +94,8 @@ public static Iterable params() { items.add( new TestCase( "2022-01-01T01:00:01Z", - "XsFI2ajcFfi45iV3AAABfhMmjmg", - "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "XsFI2S5yHCiDlfM-AAABfhMmjmg", + "lIs2K8LG_eENNC1Z3qoZTAaVP_l7r81gY5lZQumYwb0gCVmU", "2022-01-01T01:00:01.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:01Z"); @@ -106,8 +106,8 @@ public static Iterable params() { items.add( new TestCase( "1970-01-01T00:00:00Z", - "XsFI2ajcFfi45iV3AAAAAAAAAAA", - "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "XsFI2S5yHCiDlfM-AAAAAAAAAAA", + "lIs2K8LG_eENNC1Z3qoZTAaVP_l7r81gY5lZQumYwb0gCVmU", "1970-01-01T00:00:00.000Z", b -> { b.field("@timestamp", "1970-01-01T00:00:00Z"); @@ -118,8 +118,8 @@ public static Iterable params() { items.add( new TestCase( "-9998-01-01T00:00:00Z", - "XsFI2ajcFfi45iV3__6oggRgGAA", - "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "XsFI2S5yHCiDlfM-__6oggRgGAA", + "lIs2K8LG_eENNC1Z3qoZTAaVP_l7r81gY5lZQumYwb0gCVmU", "-9998-01-01T00:00:00.000Z", b -> { b.field("@timestamp", "-9998-01-01T00:00:00Z"); @@ -130,8 +130,8 @@ public static Iterable params() { items.add( new TestCase( "9998-01-01T00:00:00Z", - "XsFI2ajcFfi45iV3AADmaSK9hAA", - "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "XsFI2S5yHCiDlfM-AADmaSK9hAA", + "lIs2K8LG_eENNC1Z3qoZTAaVP_l7r81gY5lZQumYwb0gCVmU", "9998-01-01T00:00:00.000Z", b -> { b.field("@timestamp", "9998-01-01T00:00:00Z"); @@ -144,8 +144,8 @@ public static Iterable params() { items.add( new TestCase( "r1", - "XsFI2ajcFfi45iV3AAABfhMmioA", - "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "XsFI2S5yHCiDlfM-AAABfhMmioA", + "lIs2K8LG_eENNC1Z3qoZTAaVP_l7r81gY5lZQumYwb0gCVmU", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -180,8 +180,8 @@ public static Iterable params() { items.add( new TestCase( "r2", - "1y-UzR0iuE1-sOQpAAABfhMmioA", - "JNY_frTR9GmCbhXgK4Y8W44GlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "1y-Uzfo-1P_bJDK2AAABfhMmioA", + "1j9-tNH0aYJuFeArhjxbjgaVP_l7r81gY5lZQumYwb0gCVmU", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -192,8 +192,8 @@ public static Iterable params() { items.add( new TestCase( "o.r3", - "zh4dcS1h1gf2J5a8AAABfhMmioA", - "JEyfZsJIp3UNyfWG-4SjKFIGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "zh4dcW73kG72NCdnAAABfhMmioA", + "TJ9mwkindQ3J9Yb7hKMoUgaVP_l7r81gY5lZQumYwb0gCVmU", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -209,8 +209,8 @@ public static Iterable params() { items.add( new TestCase( "k1=dog", - "XsFI2SrEiVgZlSsYAAABfhMmioA", - "KJQKpjU9U63jhh-eNJ1f8bipyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", + "XsFI2SC_bHbBSYpBAAABfhMmioA", + "lAqmNT1TreOGH540nV_xuKnJTTwGlT_5knGdNi2h70uyD5C_Mv6o5g", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -222,8 +222,8 @@ public static Iterable params() { items.add( new TestCase( "k1=pumpkin", - "XsFI2W8GX8-0QcFxAAABfhMmioA", - "KJQKpjU9U63jhh-eNJ1f8bibzw1JBpU_-VsHjSz5HC1yy_swPEM1iGoAAAAAAAAA", + "XsFI2XxAh7vYQakJAAABfhMmioA", + "lAqmNT1TreOGH540nV_xuJvPDUkGlT_5WweNLPkcLXLL-zA8QzWIag", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -235,8 +235,8 @@ public static Iterable params() { items.add( new TestCase( "k1=empty string", - "XsFI2cna58i6D-Q6AAABfhMmioA", - "KJQKpjU9U63jhh-eNJ1f8bhaCD7uBpU_-SWGG0Uv9tZ1mLO2gi9rC1IAAAAAAAAA", + "XsFI2fKoX0rSe3_6AAABfhMmioA", + "lAqmNT1TreOGH540nV_xuFoIPu4GlT_5JYYbRS_21nWYs7aCL2sLUg", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -248,8 +248,8 @@ public static Iterable params() { items.add( new TestCase( "k2", - "XsFI2VqlzAuv-06kAAABfhMmioA", - "KB9H-tGrL_UzqMcqXcgBtzypyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", + "XsFI2TfQGYgMHGkBAAABfhMmioA", + "H0f60asv9TOoxypdyAG3PKnJTTwGlT_5knGdNi2h70uyD5C_Mv6o5g", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -261,8 +261,8 @@ public static Iterable params() { items.add( new TestCase( "o.k3", - "XsFI2S_VhridAKDUAAABfhMmioA", - "KGXATwN7ISd1_EycFRJ9h6qpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", + "XsFI2e1YPwS9YAkjAAABfhMmioA", + "ZcBPA3shJ3X8TJwVEn2HqqnJTTwGlT_5knGdNi2h70uyD5C_Mv6o5g", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -274,8 +274,8 @@ public static Iterable params() { items.add( new TestCase( "o.r3", - "zh4dcUwfL7x__2oPAAABfhMmioA", - "KJaYZVZz8plfkEvvPBpi1EWpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", + "zh4dcbjCi5GWqNYzAAABfhMmioA", + "lphlVnPymV-QS-88GmLURanJTTwGlT_5knGdNi2h70uyD5C_Mv6o5g", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -305,8 +305,8 @@ public static Iterable params() { items.add( new TestCase( "L1=1", - "XsFI2fIe53BtV9PCAAABfhMmioA", - "KI4kVxcCLIMM2_VQGD575d-tm41vBpU_-TUExUU_bL3Puq_EBgIaLacAAAAAAAAA", + "XsFI2T10ZbvtTyt0AAABfhMmioA", + "jiRXFwIsgwzb9VAYPnvl362bjW8GlT_5NQTFRT9svc-6r8QGAhotpw", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -318,8 +318,8 @@ public static Iterable params() { items.add( new TestCase( "L1=min", - "XsFI2Qhu7hy1RoXRAAABfhMmioA", - "KI4kVxcCLIMM2_VQGD575d8caJ3TBpU_-cLpg-VnCBnhYk33HZBle6EAAAAAAAAA", + "XsFI2bgn8k5477GSAAABfhMmioA", + "jiRXFwIsgwzb9VAYPnvl3xxondMGlT_5wumD5WcIGeFiTfcdkGV7oQ", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -331,8 +331,8 @@ public static Iterable params() { items.add( new TestCase( "L2=1234", - "XsFI2QTrNu7TTpc-AAABfhMmioA", - "KI_1WxF60L0IczG5ftUCWdndcGtgBpU_-QfM2BaR0DMagIfw3TDu_mAAAAAAAAAA", + "XsFI2S6u8qCv1UL0AAABfhMmioA", + "j_VbEXrQvQhzMbl-1QJZ2d1wa2AGlT_5B8zYFpHQMxqAh_DdMO7-YA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -344,8 +344,8 @@ public static Iterable params() { items.add( new TestCase( "o.L3=max", - "zh4dcWBQI6THHqxoAAABfhMmioA", - "KN4a6QzKhzc3nwzNLuZkV51xxTOVBpU_-erUU1qSW4eJ0kP0RmAB9TEAAAAAAAAA", + "zh4dcSavha_vxz6GAAABfhMmioA", + "3hrpDMqHNzefDM0u5mRXnXHFM5UGlT_56tRTWpJbh4nSQ_RGYAH1MQ", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00.000Z"); @@ -375,8 +375,8 @@ public static Iterable params() { items.add( new TestCase( "i1=1", - "XsFI2UMS_RWRoHYjAAABfhMmioA", - "KLGFpvAV8QkWSmX54kXFMgitm41vBpU_-TUExUU_bL3Puq_EBgIaLacAAAAAAAAA", + "XsFI2fT0A5_AuFzKAAABfhMmioA", + "sYWm8BXxCRZKZfniRcUyCK2bjW8GlT_5NQTFRT9svc-6r8QGAhotpw", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -388,8 +388,8 @@ public static Iterable params() { items.add( new TestCase( "i1=min", - "XsFI2adlQM5ILoA1AAABfhMmioA", - "KLGFpvAV8QkWSmX54kXFMgjV8hFQBpU_-WG2MicRGWwJdBKWq2F4qy4AAAAAAAAA", + "XsFI2TUTTqNc5uyAAAABfhMmioA", + "sYWm8BXxCRZKZfniRcUyCNXyEVAGlT_5YbYyJxEZbAl0EparYXirLg", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -401,8 +401,8 @@ public static Iterable params() { items.add( new TestCase( "i2=1234", - "XsFI2bhxfB6J0kBFAAABfhMmioA", - "KJc4-5eN1uAlYuAknQQLUlxavn2sBpU_-UEXBjgaH1uYcbayrOhdgpcAAAAAAAAA", + "XsFI2bnAeI4UglLNAAABfhMmioA", + "lzj7l43W4CVi4CSdBAtSXFq-fawGlT_5QRcGOBofW5hxtrKs6F2Clw", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -414,8 +414,8 @@ public static Iterable params() { items.add( new TestCase( "o.i3=max", - "zh4dcelxKf19CbfdAAABfhMmioA", - "KKqnzPNBe8ObksSo8rNaIFPZPCcBBpU_-Rhd_U6Jn2pjQz2zpmBuJb4AAAAAAAAA", + "zh4dcZVYlwGWElFBAAABfhMmioA", + "qqfM80F7w5uSxKjys1ogU9k8JwEGlT_5GF39TomfamNDPbOmYG4lvg", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -445,8 +445,8 @@ public static Iterable params() { items.add( new TestCase( "s1=1", - "XsFI2Y_y-8kD_BFeAAABfhMmioA", - "KFi_JDbvzWyAawmh8IEXedwGlT_5rZuNb-1ruHTTZhtsXRZpZRwWFocAAAAAAAAA", + "XsFI2e5Vf7PFkb9LAAABfhMmioA", + "WL8kNu_NbIBrCaHwgRd53AaVP_mtm41v7Wu4dNNmG2xdFmllHBYWhw", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -458,8 +458,8 @@ public static Iterable params() { items.add( new TestCase( "s1=min", - "XsFI2WV8VNVnmPVNAAABfhMmioA", - "KFi_JDbvzWyAawmh8IEXedwGlT_5JgBZj9BSCms2_jgeFFhsmDlNFdMAAAAAAAAA", + "XsFI2dHghN3s9f1cAAABfhMmioA", + "WL8kNu_NbIBrCaHwgRd53AaVP_kmAFmP0FIKazb-OB4UWGyYOU0V0w", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -471,8 +471,8 @@ public static Iterable params() { items.add( new TestCase( "s2=1234", - "XsFI2VO8mUr-J5CpAAABfhMmioA", - "KKEQ2p3CkpMH61hNk_SuvI0GlT_53XBrYP5TPdmCR-vREPnt20e9f9wAAAAAAAAA", + "XsFI2eU52c3ww-79AAABfhMmioA", + "oRDancKSkwfrWE2T9K68jQaVP_ndcGtg_lM92YJH69EQ-e3bR71_3A", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -484,8 +484,8 @@ public static Iterable params() { items.add( new TestCase( "o.s3=max", - "zh4dcQKh6K11zWeuAAABfhMmioA", - "KKVMoT_-GS95fvIBtR7XK9oGlT_5Dme9-H3sen0WZ7leJpCj7-vXau4AAAAAAAAA", + "zh4dcYaxM2mQy-WyAAABfhMmioA", + "pUyhP_4ZL3l-8gG1Htcr2gaVP_kOZ734fex6fRZnuV4mkKPv69dq7g", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -515,8 +515,8 @@ public static Iterable params() { items.add( new TestCase( "b1=1", - "XsFI2dKxqgT5JDQfAAABfhMmioA", - "KGPAUhTjWOsRfDmYp3SUELatm41vBpU_-TUExUU_bL3Puq_EBgIaLacAAAAAAAAA", + "XsFI2f86PdSxFW_BAAABfhMmioA", + "Y8BSFONY6xF8OZindJQQtq2bjW8GlT_5NQTFRT9svc-6r8QGAhotpw", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -528,8 +528,8 @@ public static Iterable params() { items.add( new TestCase( "b1=min", - "XsFI2d_PD--DgUvoAAABfhMmioA", - "KGPAUhTjWOsRfDmYp3SUELYoK6qHBpU_-d8HkZFJ3aL2ZV1lgHAjT1gAAAAAAAAA", + "XsFI2TUkLTLOLx3OAAABfhMmioA", + "Y8BSFONY6xF8OZindJQQtigrqocGlT_53weRkUndovZlXWWAcCNPWA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -541,8 +541,8 @@ public static Iterable params() { items.add( new TestCase( "b2=12", - "XsFI2aqX5QjiuhsEAAABfhMmioA", - "KA58oUMzXeX1V5rh51Ste0K5K9vPBpU_-Wn8JQplO-x3CgoslYO5VksAAAAAAAAA", + "XsFI2bCS8HbUffZ-AAABfhMmioA", + "DnyhQzNd5fVXmuHnVK17Qrkr288GlT_5afwlCmU77HcKCiyVg7lWSw", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -554,8 +554,8 @@ public static Iterable params() { items.add( new TestCase( "o.s3=max", - "zh4dccJ4YtN_21XHAAABfhMmioA", - "KIwZH-StJBobjk9tCV-0OgjKmuwGBpU_-Sd-SdnoH3sbfKLgse-briEAAAAAAAAA", + "zh4dcYE4JNpL4t_4AAABfhMmioA", + "jBkf5K0kGhuOT20JX7Q6CMqa7AYGlT_5J35J2egfext8ouCx75uuIQ", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -585,8 +585,8 @@ public static Iterable params() { items.add( new TestCase( "ip1=192.168.0.1", - "XsFI2T5km9raIz_rAAABfhMmioA", - "KNj6cLPRNEkqdjfOPIbg0wULrOlWBpU_-efWDsz6B6AnnwbZ7GeeocEAAAAAAAAA", + "XsFI2UeTkcMPSivHAAABfhMmioA", + "2Ppws9E0SSp2N848huDTBQus6VYGlT_559YOzPoHoCefBtnsZ56hwQ", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -602,8 +602,8 @@ public static Iterable params() { items.add( new TestCase( "ip1=12.12.45.254", - "XsFI2QWfEH_e_6wIAAABfhMmioA", - "KNj6cLPRNEkqdjfOPIbg0wVhJ08TBpU_-bANzLhvKPczlle7Pq0z8QwAAAAAAAAA", + "XsFI2eovUFDajzh-AAABfhMmioA", + "2Ppws9E0SSp2N848huDTBWEnTxMGlT_5sA3MuG8o9zOWV7s-rTPxDA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -619,8 +619,8 @@ public static Iterable params() { items.add( new TestCase( "ip2=FE80:CD00:0000:0CDE:1257:0000:211E:729C", - "XsFI2WrrLHr1O4iQAAABfhMmioA", - "KNDo3zGxO9HfN9XYJwKw2Z20h-WsBpU_-f4dSOLGSRlL1hoY2mgERuoAAAAAAAAA", + "XsFI2coO800yZDtEAAABfhMmioA", + "0OjfMbE70d831dgnArDZnbSH5awGlT_5_h1I4sZJGUvWGhjaaARG6g", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -632,8 +632,8 @@ public static Iterable params() { items.add( new TestCase( "o.ip3=2001:db8:85a3:8d3:1319:8a2e:370:7348", - "zh4dca7d-9aKOS1MAAABfhMmioA", - "KLXDcBBWJAjgJvjSdF_EJwraAQUzBpU_-ba6HZsIyKnGcbmc3KRLlmIAAAAAAAAA", + "zh4dcVcvZNXO8eMqAAABfhMmioA", + "tcNwEFYkCOAm-NJ0X8QnCtoBBTMGlT_5trodmwjIqcZxuZzcpEuWYg", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -663,8 +663,8 @@ public static Iterable params() { items.add( new TestCase( "huge", - "WZKJR_dECvXBSl3xAAABfhMmioA", - "LIe18i0rRU_Bt9vB82F46LaS9mrUkvZq1K_2Gi7UEFMhFwNXrLA_H8TLpUr4AAAAAAAAAAAAAAA", + "WZKJR55_Xxk3QYCwAAABfhMmioA", + "h7XyLStFT8G328HzYXjotpL2atSS9mrUr_YaLtQQUyEXA1essD8fxMulSvg", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); From 2d54146818bbb6196da5ce902336285549aca248 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 29 Nov 2023 09:38:12 +0100 Subject: [PATCH 065/125] fix: TsidExtractingIdFieldMapperTests and tsid to BytesRef --- .../index/mapper/TimeSeriesIdFieldMapper.java | 2 +- .../TsidExtractingIdFieldMapperTests.java | 140 +++++++++--------- 2 files changed, 71 insertions(+), 71 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 5e2b6472a8863..ece682bb278c3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -254,7 +254,7 @@ public BytesReference withHash() throws IOException { assert tsidHashIndex == tsidHash.length; try (BytesStreamOutput out = new BytesStreamOutput(tsidHash.length)) { - out.write(tsidHash, 0, tsidHash.length); + out.writeBytesRef(new BytesRef(tsidHash, 0, tsidHash.length)); return out.bytes(); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index d0b63141f7edd..199ff2e52f803 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -82,8 +82,8 @@ public static Iterable params() { items.add( new TestCase( "2022-01-01T01:00:00Z", - "XsFI2S5yHCiDlfM-AAABfhMmioA", - "lIs2K8LG_eENNC1Z3qoZTAaVP_l7r81gY5lZQumYwb0gCVmU", + "XsFI2ajcFfi45iV3AAABfhMmioA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -94,8 +94,8 @@ public static Iterable params() { items.add( new TestCase( "2022-01-01T01:00:01Z", - "XsFI2S5yHCiDlfM-AAABfhMmjmg", - "lIs2K8LG_eENNC1Z3qoZTAaVP_l7r81gY5lZQumYwb0gCVmU", + "XsFI2ajcFfi45iV3AAABfhMmjmg", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", "2022-01-01T01:00:01.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:01Z"); @@ -106,8 +106,8 @@ public static Iterable params() { items.add( new TestCase( "1970-01-01T00:00:00Z", - "XsFI2S5yHCiDlfM-AAAAAAAAAAA", - "lIs2K8LG_eENNC1Z3qoZTAaVP_l7r81gY5lZQumYwb0gCVmU", + "XsFI2ajcFfi45iV3AAAAAAAAAAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", "1970-01-01T00:00:00.000Z", b -> { b.field("@timestamp", "1970-01-01T00:00:00Z"); @@ -118,8 +118,8 @@ public static Iterable params() { items.add( new TestCase( "-9998-01-01T00:00:00Z", - "XsFI2S5yHCiDlfM-__6oggRgGAA", - "lIs2K8LG_eENNC1Z3qoZTAaVP_l7r81gY5lZQumYwb0gCVmU", + "XsFI2ajcFfi45iV3__6oggRgGAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", "-9998-01-01T00:00:00.000Z", b -> { b.field("@timestamp", "-9998-01-01T00:00:00Z"); @@ -130,8 +130,8 @@ public static Iterable params() { items.add( new TestCase( "9998-01-01T00:00:00Z", - "XsFI2S5yHCiDlfM-AADmaSK9hAA", - "lIs2K8LG_eENNC1Z3qoZTAaVP_l7r81gY5lZQumYwb0gCVmU", + "XsFI2ajcFfi45iV3AADmaSK9hAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", "9998-01-01T00:00:00.000Z", b -> { b.field("@timestamp", "9998-01-01T00:00:00Z"); @@ -144,8 +144,8 @@ public static Iterable params() { items.add( new TestCase( "r1", - "XsFI2S5yHCiDlfM-AAABfhMmioA", - "lIs2K8LG_eENNC1Z3qoZTAaVP_l7r81gY5lZQumYwb0gCVmU", + "XsFI2ajcFfi45iV3AAABfhMmioA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -180,8 +180,8 @@ public static Iterable params() { items.add( new TestCase( "r2", - "1y-Uzfo-1P_bJDK2AAABfhMmioA", - "1j9-tNH0aYJuFeArhjxbjgaVP_l7r81gY5lZQumYwb0gCVmU", + "1y-UzR0iuE1-sOQpAAABfhMmioA", + "JNY_frTR9GmCbhXgK4Y8W44GlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -192,8 +192,8 @@ public static Iterable params() { items.add( new TestCase( "o.r3", - "zh4dcW73kG72NCdnAAABfhMmioA", - "TJ9mwkindQ3J9Yb7hKMoUgaVP_l7r81gY5lZQumYwb0gCVmU", + "zh4dcS1h1gf2J5a8AAABfhMmioA", + "JEyfZsJIp3UNyfWG-4SjKFIGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -209,8 +209,8 @@ public static Iterable params() { items.add( new TestCase( "k1=dog", - "XsFI2SC_bHbBSYpBAAABfhMmioA", - "lAqmNT1TreOGH540nV_xuKnJTTwGlT_5knGdNi2h70uyD5C_Mv6o5g", + "XsFI2SrEiVgZlSsYAAABfhMmioA", + "KJQKpjU9U63jhh-eNJ1f8bipyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -222,8 +222,8 @@ public static Iterable params() { items.add( new TestCase( "k1=pumpkin", - "XsFI2XxAh7vYQakJAAABfhMmioA", - "lAqmNT1TreOGH540nV_xuJvPDUkGlT_5WweNLPkcLXLL-zA8QzWIag", + "XsFI2W8GX8-0QcFxAAABfhMmioA", + "KJQKpjU9U63jhh-eNJ1f8bibzw1JBpU_-VsHjSz5HC1yy_swPEM1iGoAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -235,8 +235,8 @@ public static Iterable params() { items.add( new TestCase( "k1=empty string", - "XsFI2fKoX0rSe3_6AAABfhMmioA", - "lAqmNT1TreOGH540nV_xuFoIPu4GlT_5JYYbRS_21nWYs7aCL2sLUg", + "XsFI2cna58i6D-Q6AAABfhMmioA", + "KJQKpjU9U63jhh-eNJ1f8bhaCD7uBpU_-SWGG0Uv9tZ1mLO2gi9rC1IAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -248,8 +248,8 @@ public static Iterable params() { items.add( new TestCase( "k2", - "XsFI2TfQGYgMHGkBAAABfhMmioA", - "H0f60asv9TOoxypdyAG3PKnJTTwGlT_5knGdNi2h70uyD5C_Mv6o5g", + "XsFI2VqlzAuv-06kAAABfhMmioA", + "KB9H-tGrL_UzqMcqXcgBtzypyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -261,8 +261,8 @@ public static Iterable params() { items.add( new TestCase( "o.k3", - "XsFI2e1YPwS9YAkjAAABfhMmioA", - "ZcBPA3shJ3X8TJwVEn2HqqnJTTwGlT_5knGdNi2h70uyD5C_Mv6o5g", + "XsFI2S_VhridAKDUAAABfhMmioA", + "KGXATwN7ISd1_EycFRJ9h6qpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -274,8 +274,8 @@ public static Iterable params() { items.add( new TestCase( "o.r3", - "zh4dcbjCi5GWqNYzAAABfhMmioA", - "lphlVnPymV-QS-88GmLURanJTTwGlT_5knGdNi2h70uyD5C_Mv6o5g", + "zh4dcUwfL7x__2oPAAABfhMmioA", + "KJaYZVZz8plfkEvvPBpi1EWpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -305,8 +305,8 @@ public static Iterable params() { items.add( new TestCase( "L1=1", - "XsFI2T10ZbvtTyt0AAABfhMmioA", - "jiRXFwIsgwzb9VAYPnvl362bjW8GlT_5NQTFRT9svc-6r8QGAhotpw", + "XsFI2fIe53BtV9PCAAABfhMmioA", + "KI4kVxcCLIMM2_VQGD575d-tm41vBpU_-TUExUU_bL3Puq_EBgIaLacAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -318,8 +318,8 @@ public static Iterable params() { items.add( new TestCase( "L1=min", - "XsFI2bgn8k5477GSAAABfhMmioA", - "jiRXFwIsgwzb9VAYPnvl3xxondMGlT_5wumD5WcIGeFiTfcdkGV7oQ", + "XsFI2Qhu7hy1RoXRAAABfhMmioA", + "KI4kVxcCLIMM2_VQGD575d8caJ3TBpU_-cLpg-VnCBnhYk33HZBle6EAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -331,8 +331,8 @@ public static Iterable params() { items.add( new TestCase( "L2=1234", - "XsFI2S6u8qCv1UL0AAABfhMmioA", - "j_VbEXrQvQhzMbl-1QJZ2d1wa2AGlT_5B8zYFpHQMxqAh_DdMO7-YA", + "XsFI2QTrNu7TTpc-AAABfhMmioA", + "KI_1WxF60L0IczG5ftUCWdndcGtgBpU_-QfM2BaR0DMagIfw3TDu_mAAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -344,8 +344,8 @@ public static Iterable params() { items.add( new TestCase( "o.L3=max", - "zh4dcSavha_vxz6GAAABfhMmioA", - "3hrpDMqHNzefDM0u5mRXnXHFM5UGlT_56tRTWpJbh4nSQ_RGYAH1MQ", + "zh4dcWBQI6THHqxoAAABfhMmioA", + "KN4a6QzKhzc3nwzNLuZkV51xxTOVBpU_-erUU1qSW4eJ0kP0RmAB9TEAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00.000Z"); @@ -375,8 +375,8 @@ public static Iterable params() { items.add( new TestCase( "i1=1", - "XsFI2fT0A5_AuFzKAAABfhMmioA", - "sYWm8BXxCRZKZfniRcUyCK2bjW8GlT_5NQTFRT9svc-6r8QGAhotpw", + "XsFI2UMS_RWRoHYjAAABfhMmioA", + "KLGFpvAV8QkWSmX54kXFMgitm41vBpU_-TUExUU_bL3Puq_EBgIaLacAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -388,8 +388,8 @@ public static Iterable params() { items.add( new TestCase( "i1=min", - "XsFI2TUTTqNc5uyAAAABfhMmioA", - "sYWm8BXxCRZKZfniRcUyCNXyEVAGlT_5YbYyJxEZbAl0EparYXirLg", + "XsFI2adlQM5ILoA1AAABfhMmioA", + "KLGFpvAV8QkWSmX54kXFMgjV8hFQBpU_-WG2MicRGWwJdBKWq2F4qy4AAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -401,8 +401,8 @@ public static Iterable params() { items.add( new TestCase( "i2=1234", - "XsFI2bnAeI4UglLNAAABfhMmioA", - "lzj7l43W4CVi4CSdBAtSXFq-fawGlT_5QRcGOBofW5hxtrKs6F2Clw", + "XsFI2bhxfB6J0kBFAAABfhMmioA", + "KJc4-5eN1uAlYuAknQQLUlxavn2sBpU_-UEXBjgaH1uYcbayrOhdgpcAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -414,8 +414,8 @@ public static Iterable params() { items.add( new TestCase( "o.i3=max", - "zh4dcZVYlwGWElFBAAABfhMmioA", - "qqfM80F7w5uSxKjys1ogU9k8JwEGlT_5GF39TomfamNDPbOmYG4lvg", + "zh4dcelxKf19CbfdAAABfhMmioA", + "KKqnzPNBe8ObksSo8rNaIFPZPCcBBpU_-Rhd_U6Jn2pjQz2zpmBuJb4AAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -445,8 +445,8 @@ public static Iterable params() { items.add( new TestCase( "s1=1", - "XsFI2e5Vf7PFkb9LAAABfhMmioA", - "WL8kNu_NbIBrCaHwgRd53AaVP_mtm41v7Wu4dNNmG2xdFmllHBYWhw", + "XsFI2Y_y-8kD_BFeAAABfhMmioA", + "KFi_JDbvzWyAawmh8IEXedwGlT_5rZuNb-1ruHTTZhtsXRZpZRwWFocAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -458,8 +458,8 @@ public static Iterable params() { items.add( new TestCase( "s1=min", - "XsFI2dHghN3s9f1cAAABfhMmioA", - "WL8kNu_NbIBrCaHwgRd53AaVP_kmAFmP0FIKazb-OB4UWGyYOU0V0w", + "XsFI2WV8VNVnmPVNAAABfhMmioA", + "KFi_JDbvzWyAawmh8IEXedwGlT_5JgBZj9BSCms2_jgeFFhsmDlNFdMAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -471,8 +471,8 @@ public static Iterable params() { items.add( new TestCase( "s2=1234", - "XsFI2eU52c3ww-79AAABfhMmioA", - "oRDancKSkwfrWE2T9K68jQaVP_ndcGtg_lM92YJH69EQ-e3bR71_3A", + "XsFI2VO8mUr-J5CpAAABfhMmioA", + "KKEQ2p3CkpMH61hNk_SuvI0GlT_53XBrYP5TPdmCR-vREPnt20e9f9wAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -484,8 +484,8 @@ public static Iterable params() { items.add( new TestCase( "o.s3=max", - "zh4dcYaxM2mQy-WyAAABfhMmioA", - "pUyhP_4ZL3l-8gG1Htcr2gaVP_kOZ734fex6fRZnuV4mkKPv69dq7g", + "zh4dcQKh6K11zWeuAAABfhMmioA", + "KKVMoT_-GS95fvIBtR7XK9oGlT_5Dme9-H3sen0WZ7leJpCj7-vXau4AAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -515,8 +515,8 @@ public static Iterable params() { items.add( new TestCase( "b1=1", - "XsFI2f86PdSxFW_BAAABfhMmioA", - "Y8BSFONY6xF8OZindJQQtq2bjW8GlT_5NQTFRT9svc-6r8QGAhotpw", + "XsFI2dKxqgT5JDQfAAABfhMmioA", + "KGPAUhTjWOsRfDmYp3SUELatm41vBpU_-TUExUU_bL3Puq_EBgIaLacAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -528,8 +528,8 @@ public static Iterable params() { items.add( new TestCase( "b1=min", - "XsFI2TUkLTLOLx3OAAABfhMmioA", - "Y8BSFONY6xF8OZindJQQtigrqocGlT_53weRkUndovZlXWWAcCNPWA", + "XsFI2d_PD--DgUvoAAABfhMmioA", + "KGPAUhTjWOsRfDmYp3SUELYoK6qHBpU_-d8HkZFJ3aL2ZV1lgHAjT1gAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -541,8 +541,8 @@ public static Iterable params() { items.add( new TestCase( "b2=12", - "XsFI2bCS8HbUffZ-AAABfhMmioA", - "DnyhQzNd5fVXmuHnVK17Qrkr288GlT_5afwlCmU77HcKCiyVg7lWSw", + "XsFI2aqX5QjiuhsEAAABfhMmioA", + "KA58oUMzXeX1V5rh51Ste0K5K9vPBpU_-Wn8JQplO-x3CgoslYO5VksAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -554,8 +554,8 @@ public static Iterable params() { items.add( new TestCase( "o.s3=max", - "zh4dcYE4JNpL4t_4AAABfhMmioA", - "jBkf5K0kGhuOT20JX7Q6CMqa7AYGlT_5J35J2egfext8ouCx75uuIQ", + "zh4dccJ4YtN_21XHAAABfhMmioA", + "KIwZH-StJBobjk9tCV-0OgjKmuwGBpU_-Sd-SdnoH3sbfKLgse-briEAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -585,8 +585,8 @@ public static Iterable params() { items.add( new TestCase( "ip1=192.168.0.1", - "XsFI2UeTkcMPSivHAAABfhMmioA", - "2Ppws9E0SSp2N848huDTBQus6VYGlT_559YOzPoHoCefBtnsZ56hwQ", + "XsFI2T5km9raIz_rAAABfhMmioA", + "KNj6cLPRNEkqdjfOPIbg0wULrOlWBpU_-efWDsz6B6AnnwbZ7GeeocEAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -602,8 +602,8 @@ public static Iterable params() { items.add( new TestCase( "ip1=12.12.45.254", - "XsFI2eovUFDajzh-AAABfhMmioA", - "2Ppws9E0SSp2N848huDTBWEnTxMGlT_5sA3MuG8o9zOWV7s-rTPxDA", + "XsFI2QWfEH_e_6wIAAABfhMmioA", + "KNj6cLPRNEkqdjfOPIbg0wVhJ08TBpU_-bANzLhvKPczlle7Pq0z8QwAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -619,8 +619,8 @@ public static Iterable params() { items.add( new TestCase( "ip2=FE80:CD00:0000:0CDE:1257:0000:211E:729C", - "XsFI2coO800yZDtEAAABfhMmioA", - "0OjfMbE70d831dgnArDZnbSH5awGlT_5_h1I4sZJGUvWGhjaaARG6g", + "XsFI2WrrLHr1O4iQAAABfhMmioA", + "KNDo3zGxO9HfN9XYJwKw2Z20h-WsBpU_-f4dSOLGSRlL1hoY2mgERuoAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -632,8 +632,8 @@ public static Iterable params() { items.add( new TestCase( "o.ip3=2001:db8:85a3:8d3:1319:8a2e:370:7348", - "zh4dcVcvZNXO8eMqAAABfhMmioA", - "tcNwEFYkCOAm-NJ0X8QnCtoBBTMGlT_5trodmwjIqcZxuZzcpEuWYg", + "zh4dca7d-9aKOS1MAAABfhMmioA", + "KLXDcBBWJAjgJvjSdF_EJwraAQUzBpU_-ba6HZsIyKnGcbmc3KRLlmIAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -663,8 +663,8 @@ public static Iterable params() { items.add( new TestCase( "huge", - "WZKJR55_Xxk3QYCwAAABfhMmioA", - "h7XyLStFT8G328HzYXjotpL2atSS9mrUr_YaLtQQUyEXA1essD8fxMulSvg", + "WZKJR_dECvXBSl3xAAABfhMmioA", + "LIe18i0rRU_Bt9vB82F46LaS9mrUkvZq1K_2Gi7UEFMhFwNXrLA_H8TLpUr4AAAAAAAAAAAAAAA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); From 5e9cce4a1eead6cddb203e5b3acd1b42d3473690 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 29 Nov 2023 13:26:22 +0100 Subject: [PATCH 066/125] fix: Base 64 using larger buffer and new tsid values --- .../metrics/geoline-aggregation.asciidoc | 12 +-- .../rest-api-spec/test/delete/70_tsdb.yml | 8 +- .../rest-api-spec/test/tsdb/100_composite.yml | 16 ++-- .../test/tsdb/140_routing_path.yml | 17 ++-- .../test/tsdb/25_id_generation.yml | 85 ++++++++++++------- .../rest-api-spec/test/tsdb/30_snapshot.yml | 2 +- .../rest-api-spec/test/tsdb/40_search.yml | 24 +++--- .../rest-api-spec/test/tsdb/50_alias.yml | 10 +-- .../test/tsdb/60_add_dimensions.yml | 10 +-- .../test/tsdb/70_dimension_types.yml | 64 +++++++------- .../test/tsdb/80_index_resize.yml | 4 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 8 +- .../elasticsearch/search/DocValueFormat.java | 17 +++- .../TsidExtractingIdFieldMapperTests.java | 70 +++++++-------- .../rest-api-spec/test/analytics/100_tsdb.yml | 6 +- .../test/analytics/reset_tracking_rate.yml | 2 +- .../test/spatial/120_position_geo_line.yml | 60 ++++++------- 17 files changed, 229 insertions(+), 186 deletions(-) diff --git a/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc b/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc index cef25fdf401f0..4d90ec75a3dc3 100644 --- a/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc @@ -347,8 +347,8 @@ This query will result in: "aggregations": { "path": { "buckets": { - "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0": { - "key": "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0", + "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": { + "key": "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA", "doc_count": 2, "museum_tour": { "type": "Feature", @@ -361,8 +361,8 @@ This query will result in: } } }, - "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A": { - "key": "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A", + "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": { + "key": "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA", "doc_count": 5, "museum_tour": { "type": "Feature", @@ -375,8 +375,8 @@ This query will result in: } } }, - "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH": { - "key": "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH", + "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": { + "key": "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw", "doc_count": 3, "museum_tour": { "type": "Feature", diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml index 2fe56ee00614c..ce61e0a0711d0 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml @@ -43,15 +43,15 @@ location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhKULvxRQ9efYAAABiHD35_g } + - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } - match: { result: created } - match: { _version: 1 } - do: delete: index: weather_sensors - id: crxuhKULvxRQ9efYAAABiHD35_g - - match: { _id: crxuhKULvxRQ9efYAAABiHD35_g } + id: crxuhAep5Npwt_etAAABiHD35_g + - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } - match: { result: deleted } - match: { _version: 2 } @@ -68,6 +68,6 @@ location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhKULvxRQ9efYAAABiHD35_g } + - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } - match: { result: created } - match: { _version: 3 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml index 18c1ebe5130b6..3da8d8fa341c7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml @@ -92,23 +92,23 @@ composite aggregation on tsid: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key.tsid: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA" } + - match: { aggregations.tsids.buckets.0.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } - match: { aggregations.tsids.buckets.0.key.date: 1619635800000} - match: { aggregations.tsids.buckets.0.doc_count: 3 } - - match: { aggregations.tsids.buckets.1.key.tsid: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA" } + - match: { aggregations.tsids.buckets.1.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ" } - match: { aggregations.tsids.buckets.1.key.date: 1619635860000} - match: { aggregations.tsids.buckets.1.doc_count: 1 } - - match: { aggregations.tsids.buckets.2.key.tsid: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg" } + - match: { aggregations.tsids.buckets.2.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.buckets.2.key.date: 1619635800000 } - match: { aggregations.tsids.buckets.2.doc_count: 3 } - - match: { aggregations.tsids.buckets.3.key.tsid: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg" } + - match: { aggregations.tsids.buckets.3.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.buckets.3.key.date: 1619635860000} - match: { aggregations.tsids.buckets.3.doc_count: 1 } - - match: { aggregations.tsids.after_key.tsid: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg" } + - match: { aggregations.tsids.after_key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.after_key.date: 1619635860000} --- @@ -139,17 +139,17 @@ composite aggregation on tsid with after: } ] after: { - tsid: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg", + tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o", date: 1619635800000 } - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 1 } - - match: { aggregations.tsids.buckets.0.key.tsid: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg" } + - match: { aggregations.tsids.buckets.0.key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.buckets.0.key.date: 1619635860000} - match: { aggregations.tsids.buckets.0.doc_count: 1 } - - match: { aggregations.tsids.after_key.tsid: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg" } + - match: { aggregations.tsids.after_key.tsid: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" } - match: { aggregations.tsids.after_key.date: 1619635860000} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 662e6e82a0fd8..275bd446d3996 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -71,22 +71,21 @@ missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key: "M4NGZuPu-7ista-OslulS5oRP-85ihHDoibMoRwXiwRoYqddJop8zQ" } + - match: { aggregations.tsids.buckets.0.key: "JNy0BQX41tKNa3KEdjReXM85ihHDIG1DaFBdVI_fYOQvJgKOvg" } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "M4NGZuPu-7ista-OslulS9xRYkR12oDhJ6jyNsEEEXHALSWt58JvoA" } + - match: { aggregations.tsids.buckets.1.key: "JNy0BQX41tKNa3KEdjReXM912oDh9NI69d0Kk5TQ6CAdewYP5A" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.15, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key: "3LQFBfjW0o1rcoR2NF5czzmKEcMgbUNoUF1Uj99g5C8mAo6-" } + - match: { aggregations.tsids.buckets.2.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key: "3LQFBfjW0o1rcoR2NF5cz3XagOH00jr13QqTlNDoIB17Bg_k" } + - match: { aggregations.tsids.buckets.3.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 7.15 - , error: 0.01 }} + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 7.30, error: 0.01 }} --- missing dimension on routing path field: @@ -199,10 +198,10 @@ multi-value routing path field: - match: {hits.total.value: 4} - length: {aggregations.tsids.buckets: 2} - - match: {aggregations.tsids.buckets.0.key: "M4NGZuPu-7ista-OslulS5oRP-85ihHDoibMoRwXiwRoYqddJop8zQ" } + - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" } - match: {aggregations.tsids.buckets.0.doc_count: 2 } - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "M4NGZuPu-7ista-OslulS9xRYkR12oDhJ6jyNsEEEXHALSWt58JvoA" } + - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" } - match: {aggregations.tsids.buckets.1.doc_count: 2 } - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 6cb3e2a010e39..66430e9c4b2e8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -76,7 +76,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNsyIXXGJG3XgFAAABeRnS7fM} + - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} - do: bulk: @@ -85,7 +85,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNsyIXXGJG3XgFAAABeRnS7fM} + - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} - do: search: @@ -97,39 +97,39 @@ generates a consistent id: - match: {hits.total.value: 9} - - match: { hits.hits.0._id: cn4exclok5LVWXitAAABeRnRFAY } + - match: { hits.hits.0._id: cn4excfoxSs_KdA5AAABeRnRFAY } - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z } - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.1._id: cZZNsyIXXGJG3XgFAAABeRnRGTM } + - match: { hits.hits.1._id: cZZNs7B9sSWsyrL5AAABeRnRGTM } - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:04.467Z } - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.2._id: cn4exclok5LVWXitAAABeRnRYiY } + - match: { hits.hits.2._id: cn4excfoxSs_KdA5AAABeRnRYiY } - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:23.142Z } - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.3._id: cZZNsyIXXGJG3XgFAAABeRnRZ1M } + - match: { hits.hits.3._id: cZZNs7B9sSWsyrL5AAABeRnRZ1M } - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:24.467Z } - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.4._id: cZZNsyIXXGJG3XgFAAABeRnRtXM } + - match: { hits.hits.4._id: cZZNs7B9sSWsyrL5AAABeRnRtXM } - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:44.467Z } - match: { hits.hits.4._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.5._id: cn4exclok5LVWXitAAABeRnR11Y } + - match: { hits.hits.5._id: cn4excfoxSs_KdA5AAABeRnR11Y } - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:50:53.142Z } - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.6._id: cn4exclok5LVWXitAAABeRnR_mY } + - match: { hits.hits.6._id: cn4excfoxSs_KdA5AAABeRnR_mY } - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:03.142Z } - match: { hits.hits.6._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.7._id: cZZNsyIXXGJG3XgFAAABeRnSA5M } + - match: { hits.hits.7._id: cZZNs7B9sSWsyrL5AAABeRnSA5M } - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:51:04.467Z } - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.8._id: cZZNsyIXXGJG3XgFAAABeRnS7fM } + - match: { hits.hits.8._id: cZZNs7B9sSWsyrL5AAABeRnS7fM } - match: { hits.hits.8._source.@timestamp: 2021-04-28T18:52:04.467Z } - match: { hits.hits.8._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } @@ -170,7 +170,7 @@ index a new document on top of an old one: network: tx: 111434595272 rx: 430605511 - - match: {_id: cn4exclok5LVWXitAAABeRnR_mY} + - match: {_id: cn4excfoxSs_KdA5AAABeRnR_mY} - do: search: @@ -215,7 +215,7 @@ index a new document on top of an old one over bulk: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: {items.0.index._id: cn4exclok5LVWXitAAABeRnR_mY} + - match: {items.0.index._id: cn4excfoxSs_KdA5AAABeRnR_mY} - do: search: @@ -239,7 +239,7 @@ create operation on top of old document fails: reason: id generation changed in 8.2 - do: - catch: "/\\[cn4exclok5LVWXitAAABeRnR_mY\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" + catch: "/\\[cn4excfoxSs_KdA5AAABeRnR_mY\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" index: refresh: true index: test @@ -268,7 +268,7 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[cn4exclok5LVWXitAAABeRnR_mY][KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[cn4excfoxSs_KdA5AAABeRnR_mY][KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: @@ -276,6 +276,15 @@ ids query: version: " - 8.11.99" reason: _tsid hasing introduced in 8.12 + - do: + search: + index: test + body: + query: + match_all: {} + + - match: { hits.total.value: 8 } + - do: search: index: test @@ -284,13 +293,13 @@ ids query: - field: k8s.pod.network.tx query: ids: - values: ["cn4exclok5LVWXitAAABeRnR11Y", "cn4exclok5LVWXitAAABeRnR_mY"] + values: ["cn4excfoxSs_KdA5AAABeRnR11Y", "cn4excfoxSs_KdA5AAABeRnR_mY"] sort: ["@timestamp"] - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "cn4exclok5LVWXitAAABeRnR11Y"} + - match: {hits.hits.0._id: "cn4excfoxSs_KdA5AAABeRnR11Y"} - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434587694]} - - match: {hits.hits.1._id: "cn4exclok5LVWXitAAABeRnR_mY" } - - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [ 1434595272 ]} + - match: {hits.hits.1._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } + - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [1434595272]} --- get: @@ -298,12 +307,21 @@ get: version: " - 8.11.99" reason: _tsid hasing introduced in 8.12 + - do: + search: + index: test + body: + query: + match_all: {} + + - match: { hits.total.value: 8 } + - do: get: index: test - id: cZZNsyIXXGJG3XgFAAABeRnSA5M + id: cZZNs7B9sSWsyrL5AAABeRnSA5M - match: {_index: test} - - match: {_id: cZZNsyIXXGJG3XgFAAABeRnSA5M} + - match: {_id: cZZNs7B9sSWsyrL5AAABeRnSA5M} - match: _source: "@timestamp": "2021-04-28T18:51:04.467Z" @@ -351,7 +369,7 @@ delete: - do: delete: index: test - id: cn4exclok5LVWXitAAABeRnR_mY + id: cn4excfoxSs_KdA5AAABeRnR_mY - match: {result: deleted} --- @@ -385,26 +403,35 @@ delete over _bulk: version: " - 8.1.99" reason: ids generation changed in 8.2 + - do: + search: + index: test + body: + query: + match_all: {} + + - match: { hits.total.value: 8 } + # mget call added to investigate test failure: https://github.com/elastic/elasticsearch/issues/93852 # (should be removed when test issue is resolved) - do: mget: index: test body: - ids: [ cn4exclok5LVWXitAAABeRnR_mY, cn4exclok5LVWXitAAABeRnR11Y ] + ids: [ cn4excfoxSs_KdA5AAABeRnR_mY, cn4excfoxSs_KdA5AAABeRnR11Y ] - match: { docs.0._index: "test" } - - match: { docs.0._id: "cn4exclok5LVWXitAAABeRnR_mY" } + - match: { docs.0._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } - match: { docs.0.found: true } - match: { docs.1._index: "test" } - - match: { docs.1._id: "cn4exclok5LVWXitAAABeRnR11Y" } + - match: { docs.1._id: "cn4excfoxSs_KdA5AAABeRnR11Y" } - match: { docs.1.found: true } - do: bulk: index: test body: - - '{"delete": {"_id": "cn4exclok5LVWXitAAABeRnR_mY"}}' - - '{"delete": {"_id": "cn4exclok5LVWXitAAABeRnR11Y"}}' + - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR_mY"}}' + - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR11Y"}}' - '{"delete": {"_id": "not found ++ not found"}}' - match: {items.0.delete.result: deleted} - match: {items.1.delete.result: deleted} @@ -454,7 +481,7 @@ routing_path matches deep object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"bar": {"baz": {"uid": "uid1"}}}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: OcEOGftorSJSldkIAAABeRnRGTM} + - match: {items.0.index._id: OcEOGchJrjH1fFX8AAABeRnRGTM} --- routing_path matches object: @@ -495,4 +522,4 @@ routing_path matches object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"uid": "uid1"}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: 8bgiqa25oA7vLNHsAAABeRnRGTM} + - match: {items.0.index._id: 8bgiqW9JKwAyp1bZAAABeRnRGTM} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 6424c74d93413..142e17e4f665c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -148,5 +148,5 @@ teardown: - match: {hits.total.value: 1} - match: {hits.hits.0._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507} - - match: {hits.hits.0.fields._tsid: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"]} + - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index 1275411194e23..beb36631c6487 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -185,7 +185,7 @@ fetch a tag: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"]} + - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} --- aggregate a dimension: @@ -282,9 +282,9 @@ aggregate a tag: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA"} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"} + - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -320,15 +320,15 @@ sort by tsid: sort: [ "_tsid", "@timestamp" ] - match: {hits.total.value: 8} - # NOTE: 98023 - Bug: the value returned by 'sort' and 'fields' API should be the same - - match: {hits.hits.0.sort: ["KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA", 1619635803142]} - - match: {hits.hits.0.fields._tsid: [ "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA"]} - - match: {hits.hits.1.sort: ["KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA", 1619635823142]} - - match: {hits.hits.1.fields._tsid: [ "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA"]} + - match: {hits.hits.0.sort: ["KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ", 1619635803142]} + - match: {hits.hits.0.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"]} - - match: {hits.hits.4.sort: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg", 1619635804467]} - - match: {hits.hits.4.fields._tsid: [ "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"]} + - match: {hits.hits.1.sort: ["KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ", 1619635823142]} + - match: {hits.hits.1.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"]} - - match: {hits.hits.7.sort: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg", 1619635864467]} - - match: {hits.hits.7.fields._tsid: [ "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"]} + - match: {hits.hits.4.sort: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o", 1619635804467]} + - match: {hits.hits.4.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} + + - match: {hits.hits.7.sort: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o", 1619635864467]} + - match: {hits.hits.7.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index fe83794064848..c1b304cb1c155 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -85,9 +85,9 @@ search an alias: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA"} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"} + - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- @@ -129,10 +129,10 @@ index into alias: _key: asc - match: {hits.total.value: 12} - - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA"} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"} + - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - - match: {aggregations.tsids.buckets.2.key: "KMQn1H8GA7xNFfZA53p2lL6UpbL76qjIUtiX4BxwB2rktJiZ2LipiA"} + - match: {aggregations.tsids.buckets.2.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.2.doc_count: 4} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index e34d9fccd0253..4f9089533ba8e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -48,7 +48,7 @@ add dimensions with put_mapping: - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ "27hcKTYkXCOfYiuRWRTWygaVP_l7r81gY5lZQumYwb0gCVmU" ] } + - match: {hits.hits.0.fields._tsid: [ "JNu4XCk2JFwjn2IrkVkU1soGlT_5e6_NYGOZWULpmMG9IAlZlA" ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -96,7 +96,7 @@ add dimensions to no dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ "27hcKTYkXCOfYiuRWRTWygaVP_l7r81gY5lZQumYwb0gCVmU" ] } + - match: {hits.hits.0.fields._tsid: [ "JNu4XCk2JFwjn2IrkVkU1soGlT_5e6_NYGOZWULpmMG9IAlZlA" ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -144,7 +144,7 @@ add dimensions to no dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ "27hcKTYkXCOfYiuRWRTWygaVP_l7r81gY5lZQumYwb0gCVmU" ] } + - match: {hits.hits.0.fields._tsid: [ "JNu4XCk2JFwjn2IrkVkU1soGlT_5e6_NYGOZWULpmMG9IAlZlA" ] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -196,7 +196,7 @@ add dimensions to some dims with dynamic_template over index: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: ["-CXfM53tOdTdJn8GSJ1F1QaVP_kGlT_5U9X7I_7uFS_raLrCjO5zpg"] } + - match: {hits.hits.0.fields._tsid: ["KPgl3zOd7TnU3SZ_BkidRdUGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y"] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} --- @@ -247,5 +247,5 @@ add dimensions to some dims with dynamic_template over bulk: - field: _tsid - field: "@timestamp" - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: ["-CXfM53tOdTdJn8GSJ1F1QaVP_kGlT_5U9X7I_7uFS_raLrCjO5zpg"] } + - match: {hits.hits.0.fields._tsid: ["KPgl3zOd7TnU3SZ_BkidRdUGlT_5BpU_-VPV-yP-7hUv62i6wozuc6Y"] } - match: {hits.hits.0.fields.@timestamp: ["2021-04-28T18:35:24.467Z"]} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index f8000e56b0f1a..74eb8ea9ad4fa 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -65,10 +65,10 @@ keyword dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "3LQFBfjW0o1rcoR2NF5czzmKEcMgbUNoUF1Uj99g5C8mAo6-"} + - match: {aggregations.tsids.buckets.0.key: "JNy0BQX41tKNa3KEdjReXM85ihHDIG1DaFBdVI_fYOQvJgKOvg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: "3LQFBfjW0o1rcoR2NF5cz3XagOH00jr13QqTlNDoIB17Bg_k"} + - match: {aggregations.tsids.buckets.1.key: "JNy0BQX41tKNa3KEdjReXM912oDh9NI69d0Kk5TQ6CAdewYP5A"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -144,19 +144,19 @@ flattened dimension: - match: { hits.total.value: 8} - length: { aggregations.tsids.buckets: 4} - - match: { aggregations.tsids.buckets.0.key: "ItgQI_4ahoh-OTREF92WDShpnb15EMdHeRDHR-MK5Mp12oDhH3K1xh2p1tygnxUG6-oW1A" } + - match: { aggregations.tsids.buckets.0.key: "NCLYECP-GoaIfjk0RBfdlg0oaZ29eRDHR3kQx0fjCuTKddqA4R9ytcYdqdbcoJ8VBuvqFtQ" } - match: { aggregations.tsids.buckets.0.doc_count: 2 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "ItgQI_4ahoh-OTREF92WDaud_up5EMdHeRDHR4sO9O45ihHDgR5QgorUoYU-bGoJZDJAkQ" } + - match: { aggregations.tsids.buckets.1.key: "NCLYECP-GoaIfjk0RBfdlg2rnf7qeRDHR3kQx0eLDvTuOYoRw4EeUIKK1KGFPmxqCWQyQJE" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key: "ItgQI_4ahoh-OTREF92WDc_WXVR5EMdHeRDHR-MK5Mp12oDhqLsKJK4LED4-EF0tg5eRVg" } + - match: { aggregations.tsids.buckets.2.key: "NCLYECP-GoaIfjk0RBfdlg3P1l1UeRDHR3kQx0fjCuTKddqA4ai7CiSuCxA-PhBdLYOXkVY" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key: "ItgQI_4ahoh-OTREF92WDeAp6XN5EMdHeRDHR4sO9O45ihHDpZqmD8_TtgiF1Xc0UkOOMQ" } + - match: { aggregations.tsids.buckets.3.key: "NCLYECP-GoaIfjk0RBfdlg3gKelzeRDHR3kQx0eLDvTuOYoRw6Wapg_P07YIhdV3NFJDjjE" } - match: { aggregations.tsids.buckets.3.doc_count: 2 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.65, error: 0.01 }} @@ -232,11 +232,11 @@ flattened empty dimension: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 2 } - - match: { aggregations.tsids.buckets.0.key: "3LQFBfjW0o1rcoR2NF5czzmKEcMgbUNoUF1Uj99g5C8mAo6-" } + - match: { aggregations.tsids.buckets.0.key: "JNy0BQX41tKNa3KEdjReXM85ihHDIG1DaFBdVI_fYOQvJgKOvg" } - match: { aggregations.tsids.buckets.0.doc_count: 4 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.69, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "3LQFBfjW0o1rcoR2NF5cz3XagOH00jr13QqTlNDoIB17Bg_k" } + - match: { aggregations.tsids.buckets.1.key: "JNy0BQX41tKNa3KEdjReXM912oDh9NI69d0Kk5TQ6CAdewYP5A" } - match: { aggregations.tsids.buckets.1.doc_count: 4 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.22, error: 0.01 }} @@ -312,29 +312,29 @@ flattened field missing routing path field: - match: { hits.total.value: 8 } - length: { aggregations.tsids.buckets: 6 } - - match: { aggregations.tsids.buckets.0.key.: "XXEWRB81VefgRtsBbneB7_FNd5soaZ29eRDHR3XagOHXoW1-8_85uSUJruBcNyZ0" } + - match: { aggregations.tsids.buckets.0.key.: "LPBzSKpPysYDR-l1jvYA8jPxTXebeRDHRzmKEcOUW9u_kzgC7pSzoi1utVcm" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "XXEWRB81VefgRtsBbneB7_FNd5urnf7qeRDHRzmKEcNTc-7NMjWYz3yBCqqKEOhk" } - - match: { aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} + - match: { aggregations.tsids.buckets.1.key: "LPBzSKpPysYDR-l1jvYA8jPxTXebeRDHR3XagOEjGQJfwIB2Q5kLDL76leH4" } + - match: { aggregations.tsids.buckets.1.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key: "XXEWRB81VefgRtsBbneB7_FNd5vP1l1UeRDHR3XagOGLVJxylWdECGmMKtaGoWN9" } - - match: { aggregations.tsids.buckets.2.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} + - match: { aggregations.tsids.buckets.2.key: "MF1xFkQfNVXn4EbbAW53ge_xTXebKGmdvXkQx0d12oDh16FtfvP_ObklCa7gXDcmdA" } + - match: { aggregations.tsids.buckets.2.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key: "XXEWRB81VefgRtsBbneB7_FNd5vgKelzeRDHRzmKEcOppLmEeAsCbQTdeQcbt92R" } - - match: { aggregations.tsids.buckets.3.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} + - match: { aggregations.tsids.buckets.3.key: "MF1xFkQfNVXn4EbbAW53ge_xTXebq53-6nkQx0c5ihHDU3PuzTI1mM98gQqqihDoZA" } + - match: { aggregations.tsids.buckets.3.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.4.key: "8HNIqk_KxgNH6XWO9gDyM_FNd5t5EMdHOYoRw5Rb27-TOALulLOiLW61VyY" } - - match: { aggregations.tsids.buckets.4.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 6.70, error: 0.01 }} + - match: { aggregations.tsids.buckets.4.key: "MF1xFkQfNVXn4EbbAW53ge_xTXebz9ZdVHkQx0d12oDhi1SccpVnRAhpjCrWhqFjfQ" } + - match: { aggregations.tsids.buckets.4.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.4.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.5.key: "8HNIqk_KxgNH6XWO9gDyM_FNd5t5EMdHddqA4SMZAl_AgHZDmQsMvvqV4fg" } + - match: { aggregations.tsids.buckets.5.key: "MF1xFkQfNVXn4EbbAW53ge_xTXeb4Cnpc3kQx0c5ihHDqaS5hHgLAm0E3XkHG7fdkQ" } - match: { aggregations.tsids.buckets.5.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.5.voltage.value: { value: 7.30, error: 0.01 }} + - close_to: { aggregations.tsids.buckets.5.voltage.value: { value: 6.60, error: 0.01 }} --- flattened field misspelled routing path field: @@ -412,19 +412,19 @@ flattened field misspelled routing path field: - match: { hits.total.value: 6 } - length: { aggregations.tsids.buckets: 4 } - - match: { aggregations.tsids.buckets.0.key: "GzT4lnbKgCD4uvDfky7fKyhpnb16BF3I14dc9I-CENXOC5pG" } + - match: { aggregations.tsids.buckets.0.key: "JBs0-JZ2yoAg-Lrw35Mu3ysoaZ29egRdyNeHXPSPghDVzguaRg" } - match: { aggregations.tsids.buckets.0.doc_count: 1 } - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} - - match: { aggregations.tsids.buckets.1.key: "GzT4lnbKgCD4uvDfky7fK6ud_uqz35VcVnqONkJuX-KJdKzt" } + - match: { aggregations.tsids.buckets.1.key: "JBs0-JZ2yoAg-Lrw35Mu3yurnf7qs9-VXFZ6jjZCbl_iiXSs7Q" } - match: { aggregations.tsids.buckets.1.doc_count: 2 } - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - match: { aggregations.tsids.buckets.2.key: "GzT4lnbKgCD4uvDfky7fK8_WXVSWaVcRBU1eselS-mftvLSM" } + - match: { aggregations.tsids.buckets.2.key: "JBs0-JZ2yoAg-Lrw35Mu3yvP1l1UlmlXEQVNXrHpUvpn7by0jA" } - match: { aggregations.tsids.buckets.2.doc_count: 2 } - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - match: { aggregations.tsids.buckets.3.key: "GzT4lnbKgCD4uvDfky7fK-Ap6XP1ZImrN5iHtrNr_vLENcxk" } + - match: { aggregations.tsids.buckets.3.key: "JBs0-JZ2yoAg-Lrw35Mu3yvgKelz9WSJqzeYh7aza_7yxDXMZA" } - match: { aggregations.tsids.buckets.3.doc_count: 1 } - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} @@ -496,10 +496,10 @@ long dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "xq55J0GFz9YiFjjGg8TYR0OAph3vei62lefsslX6BB--Wt5zs3IyGg"} + - match: {aggregations.tsids.buckets.0.key: "KMaueSdBhc_WIhY4xoPE2EdDgKYd73outpXn7LJV-gQfvlrec7NyMho"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: "xq55J0GFz9YiFjjGg8TYR62bjW_vei62ahJNwWZ9AFiM6Mxo5YznOw"} + - match: {aggregations.tsids.buckets.1.key: "KMaueSdBhc_WIhY4xoPE2Eetm41v73outmoSTcFmfQBYjOjMaOWM5zs"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} @@ -571,10 +571,10 @@ ip dimension: field: voltage - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "tWt3sTyhkvJ3619KbQfqMNjNv6_vei62iwH54Nv-fNOshWwkVVBHZw"} + - match: {aggregations.tsids.buckets.0.key: "KLVrd7E8oZLyd-tfSm0H6jDYzb-v73outosB-eDb_nzTrIVsJFVQR2c"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 3.3, error: 0.01 }} - - match: {aggregations.tsids.buckets.1.key: "tWt3sTyhkvJ3619KbQfqMNpo4Urvei62SqkbxHrkabhVSGnK5zXXPA"} + - match: {aggregations.tsids.buckets.1.key: "KLVrd7E8oZLyd-tfSm0H6jDaaOFK73outkqpG8R65Gm4VUhpyuc11zw"} - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index c710a5bca8d0a..4e3bc2510e55c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -124,7 +124,7 @@ shrink: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"]} + - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} --- clone: @@ -148,4 +148,4 @@ clone: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: ["KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"]} + - match: {hits.hits.0.fields._tsid: ["KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"]} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index ece682bb278c3..7ef29ab3f1f75 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -329,11 +329,13 @@ private void add(String fieldName, BytesReference encoded) throws IOException { } } - public static Object decodeTsid(BytesRef bytesRef) { + public static Object decodeTsid(final BytesRef bytesRef) { return base64Encode(bytesRef); } - private static String base64Encode(BytesRef bytesRef) { - return BASE64_ENCODER.encodeToString(bytesRef.bytes); + private static String base64Encode(final BytesRef bytesRef) { + byte[] bytes = new byte[bytesRef.length]; + System.arraycopy(bytesRef.bytes, 0, bytes, 0, bytesRef.length); + return BASE64_ENCODER.encodeToString(bytes); } } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 0ee2d47a6e1d6..90a2c2996dd4d 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -675,6 +675,9 @@ public double parseDouble(String value, boolean roundUp, LongSupplier now) { * DocValues format for time series id. */ class TimeSeriesIdDocValueFormat implements DocValueFormat { + + private static final Base64.Encoder BASE64_ENCODER = Base64.getUrlEncoder().withoutPadding(); + private TimeSeriesIdDocValueFormat() {} @Override @@ -690,9 +693,21 @@ public String toString() { return "tsid"; } + /** + * @param value The TSID as a {@link BytesRef} + * @return the Base 64 encoded TSID + */ @Override public Object format(BytesRef value) { - return Base64.getUrlEncoder().withoutPadding().encodeToString(value.bytes); + // NOTE: we need to do this copy to avoid encoding spurious bytes at the end of the + // BytesRef buffer. When sorting results the TermOrdValComparator will copy + // values using a BytesRefBuilder whose underlying bytes buffer is sized incorrectly, + // allocating more bytes than actually required. As a result of these additional bytes + // the Base 64 encoding might include spurious bytes (typically 0s) which result in + // additional unwanted TSID trailing characters. + byte[] bytes = new byte[value.length]; + System.arraycopy(value.bytes, 0, bytes, 0, value.length); + return BASE64_ENCODER.encodeToString(bytes); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index 199ff2e52f803..c19c21d54a569 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -83,7 +83,7 @@ public static Iterable params() { new TestCase( "2022-01-01T01:00:00Z", "XsFI2ajcFfi45iV3AAABfhMmioA", - "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -95,7 +95,7 @@ public static Iterable params() { new TestCase( "2022-01-01T01:00:01Z", "XsFI2ajcFfi45iV3AAABfhMmjmg", - "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:01.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:01Z"); @@ -107,7 +107,7 @@ public static Iterable params() { new TestCase( "1970-01-01T00:00:00Z", "XsFI2ajcFfi45iV3AAAAAAAAAAA", - "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "1970-01-01T00:00:00.000Z", b -> { b.field("@timestamp", "1970-01-01T00:00:00Z"); @@ -119,7 +119,7 @@ public static Iterable params() { new TestCase( "-9998-01-01T00:00:00Z", "XsFI2ajcFfi45iV3__6oggRgGAA", - "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "-9998-01-01T00:00:00.000Z", b -> { b.field("@timestamp", "-9998-01-01T00:00:00Z"); @@ -131,7 +131,7 @@ public static Iterable params() { new TestCase( "9998-01-01T00:00:00Z", "XsFI2ajcFfi45iV3AADmaSK9hAA", - "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "9998-01-01T00:00:00.000Z", b -> { b.field("@timestamp", "9998-01-01T00:00:00Z"); @@ -145,7 +145,7 @@ public static Iterable params() { new TestCase( "r1", "XsFI2ajcFfi45iV3AAABfhMmioA", - "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -181,7 +181,7 @@ public static Iterable params() { new TestCase( "r2", "1y-UzR0iuE1-sOQpAAABfhMmioA", - "JNY_frTR9GmCbhXgK4Y8W44GlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "JNY_frTR9GmCbhXgK4Y8W44GlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -193,7 +193,7 @@ public static Iterable params() { new TestCase( "o.r3", "zh4dcS1h1gf2J5a8AAABfhMmioA", - "JEyfZsJIp3UNyfWG-4SjKFIGlT_5e6_NYGOZWULpmMG9IAlZlAAAAAAAAAAAAAAA", + "JEyfZsJIp3UNyfWG-4SjKFIGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -210,7 +210,7 @@ public static Iterable params() { new TestCase( "k1=dog", "XsFI2SrEiVgZlSsYAAABfhMmioA", - "KJQKpjU9U63jhh-eNJ1f8bipyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", + "KJQKpjU9U63jhh-eNJ1f8bipyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -223,7 +223,7 @@ public static Iterable params() { new TestCase( "k1=pumpkin", "XsFI2W8GX8-0QcFxAAABfhMmioA", - "KJQKpjU9U63jhh-eNJ1f8bibzw1JBpU_-VsHjSz5HC1yy_swPEM1iGoAAAAAAAAA", + "KJQKpjU9U63jhh-eNJ1f8bibzw1JBpU_-VsHjSz5HC1yy_swPEM1iGo", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -236,7 +236,7 @@ public static Iterable params() { new TestCase( "k1=empty string", "XsFI2cna58i6D-Q6AAABfhMmioA", - "KJQKpjU9U63jhh-eNJ1f8bhaCD7uBpU_-SWGG0Uv9tZ1mLO2gi9rC1IAAAAAAAAA", + "KJQKpjU9U63jhh-eNJ1f8bhaCD7uBpU_-SWGG0Uv9tZ1mLO2gi9rC1I", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -249,7 +249,7 @@ public static Iterable params() { new TestCase( "k2", "XsFI2VqlzAuv-06kAAABfhMmioA", - "KB9H-tGrL_UzqMcqXcgBtzypyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", + "KB9H-tGrL_UzqMcqXcgBtzypyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -262,7 +262,7 @@ public static Iterable params() { new TestCase( "o.k3", "XsFI2S_VhridAKDUAAABfhMmioA", - "KGXATwN7ISd1_EycFRJ9h6qpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", + "KGXATwN7ISd1_EycFRJ9h6qpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -275,7 +275,7 @@ public static Iterable params() { new TestCase( "o.r3", "zh4dcUwfL7x__2oPAAABfhMmioA", - "KJaYZVZz8plfkEvvPBpi1EWpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOYAAAAAAAAA", + "KJaYZVZz8plfkEvvPBpi1EWpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -306,7 +306,7 @@ public static Iterable params() { new TestCase( "L1=1", "XsFI2fIe53BtV9PCAAABfhMmioA", - "KI4kVxcCLIMM2_VQGD575d-tm41vBpU_-TUExUU_bL3Puq_EBgIaLacAAAAAAAAA", + "KI4kVxcCLIMM2_VQGD575d-tm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -319,7 +319,7 @@ public static Iterable params() { new TestCase( "L1=min", "XsFI2Qhu7hy1RoXRAAABfhMmioA", - "KI4kVxcCLIMM2_VQGD575d8caJ3TBpU_-cLpg-VnCBnhYk33HZBle6EAAAAAAAAA", + "KI4kVxcCLIMM2_VQGD575d8caJ3TBpU_-cLpg-VnCBnhYk33HZBle6E", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -332,7 +332,7 @@ public static Iterable params() { new TestCase( "L2=1234", "XsFI2QTrNu7TTpc-AAABfhMmioA", - "KI_1WxF60L0IczG5ftUCWdndcGtgBpU_-QfM2BaR0DMagIfw3TDu_mAAAAAAAAAA", + "KI_1WxF60L0IczG5ftUCWdndcGtgBpU_-QfM2BaR0DMagIfw3TDu_mA", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -345,7 +345,7 @@ public static Iterable params() { new TestCase( "o.L3=max", "zh4dcWBQI6THHqxoAAABfhMmioA", - "KN4a6QzKhzc3nwzNLuZkV51xxTOVBpU_-erUU1qSW4eJ0kP0RmAB9TEAAAAAAAAA", + "KN4a6QzKhzc3nwzNLuZkV51xxTOVBpU_-erUU1qSW4eJ0kP0RmAB9TE", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00.000Z"); @@ -376,7 +376,7 @@ public static Iterable params() { new TestCase( "i1=1", "XsFI2UMS_RWRoHYjAAABfhMmioA", - "KLGFpvAV8QkWSmX54kXFMgitm41vBpU_-TUExUU_bL3Puq_EBgIaLacAAAAAAAAA", + "KLGFpvAV8QkWSmX54kXFMgitm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -389,7 +389,7 @@ public static Iterable params() { new TestCase( "i1=min", "XsFI2adlQM5ILoA1AAABfhMmioA", - "KLGFpvAV8QkWSmX54kXFMgjV8hFQBpU_-WG2MicRGWwJdBKWq2F4qy4AAAAAAAAA", + "KLGFpvAV8QkWSmX54kXFMgjV8hFQBpU_-WG2MicRGWwJdBKWq2F4qy4", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -402,7 +402,7 @@ public static Iterable params() { new TestCase( "i2=1234", "XsFI2bhxfB6J0kBFAAABfhMmioA", - "KJc4-5eN1uAlYuAknQQLUlxavn2sBpU_-UEXBjgaH1uYcbayrOhdgpcAAAAAAAAA", + "KJc4-5eN1uAlYuAknQQLUlxavn2sBpU_-UEXBjgaH1uYcbayrOhdgpc", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -415,7 +415,7 @@ public static Iterable params() { new TestCase( "o.i3=max", "zh4dcelxKf19CbfdAAABfhMmioA", - "KKqnzPNBe8ObksSo8rNaIFPZPCcBBpU_-Rhd_U6Jn2pjQz2zpmBuJb4AAAAAAAAA", + "KKqnzPNBe8ObksSo8rNaIFPZPCcBBpU_-Rhd_U6Jn2pjQz2zpmBuJb4", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -446,7 +446,7 @@ public static Iterable params() { new TestCase( "s1=1", "XsFI2Y_y-8kD_BFeAAABfhMmioA", - "KFi_JDbvzWyAawmh8IEXedwGlT_5rZuNb-1ruHTTZhtsXRZpZRwWFocAAAAAAAAA", + "KFi_JDbvzWyAawmh8IEXedwGlT_5rZuNb-1ruHTTZhtsXRZpZRwWFoc", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -459,7 +459,7 @@ public static Iterable params() { new TestCase( "s1=min", "XsFI2WV8VNVnmPVNAAABfhMmioA", - "KFi_JDbvzWyAawmh8IEXedwGlT_5JgBZj9BSCms2_jgeFFhsmDlNFdMAAAAAAAAA", + "KFi_JDbvzWyAawmh8IEXedwGlT_5JgBZj9BSCms2_jgeFFhsmDlNFdM", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -472,7 +472,7 @@ public static Iterable params() { new TestCase( "s2=1234", "XsFI2VO8mUr-J5CpAAABfhMmioA", - "KKEQ2p3CkpMH61hNk_SuvI0GlT_53XBrYP5TPdmCR-vREPnt20e9f9wAAAAAAAAA", + "KKEQ2p3CkpMH61hNk_SuvI0GlT_53XBrYP5TPdmCR-vREPnt20e9f9w", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -485,7 +485,7 @@ public static Iterable params() { new TestCase( "o.s3=max", "zh4dcQKh6K11zWeuAAABfhMmioA", - "KKVMoT_-GS95fvIBtR7XK9oGlT_5Dme9-H3sen0WZ7leJpCj7-vXau4AAAAAAAAA", + "KKVMoT_-GS95fvIBtR7XK9oGlT_5Dme9-H3sen0WZ7leJpCj7-vXau4", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -516,7 +516,7 @@ public static Iterable params() { new TestCase( "b1=1", "XsFI2dKxqgT5JDQfAAABfhMmioA", - "KGPAUhTjWOsRfDmYp3SUELatm41vBpU_-TUExUU_bL3Puq_EBgIaLacAAAAAAAAA", + "KGPAUhTjWOsRfDmYp3SUELatm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -529,7 +529,7 @@ public static Iterable params() { new TestCase( "b1=min", "XsFI2d_PD--DgUvoAAABfhMmioA", - "KGPAUhTjWOsRfDmYp3SUELYoK6qHBpU_-d8HkZFJ3aL2ZV1lgHAjT1gAAAAAAAAA", + "KGPAUhTjWOsRfDmYp3SUELYoK6qHBpU_-d8HkZFJ3aL2ZV1lgHAjT1g", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -542,7 +542,7 @@ public static Iterable params() { new TestCase( "b2=12", "XsFI2aqX5QjiuhsEAAABfhMmioA", - "KA58oUMzXeX1V5rh51Ste0K5K9vPBpU_-Wn8JQplO-x3CgoslYO5VksAAAAAAAAA", + "KA58oUMzXeX1V5rh51Ste0K5K9vPBpU_-Wn8JQplO-x3CgoslYO5Vks", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -555,7 +555,7 @@ public static Iterable params() { new TestCase( "o.s3=max", "zh4dccJ4YtN_21XHAAABfhMmioA", - "KIwZH-StJBobjk9tCV-0OgjKmuwGBpU_-Sd-SdnoH3sbfKLgse-briEAAAAAAAAA", + "KIwZH-StJBobjk9tCV-0OgjKmuwGBpU_-Sd-SdnoH3sbfKLgse-briE", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -586,7 +586,7 @@ public static Iterable params() { new TestCase( "ip1=192.168.0.1", "XsFI2T5km9raIz_rAAABfhMmioA", - "KNj6cLPRNEkqdjfOPIbg0wULrOlWBpU_-efWDsz6B6AnnwbZ7GeeocEAAAAAAAAA", + "KNj6cLPRNEkqdjfOPIbg0wULrOlWBpU_-efWDsz6B6AnnwbZ7GeeocE", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -603,7 +603,7 @@ public static Iterable params() { new TestCase( "ip1=12.12.45.254", "XsFI2QWfEH_e_6wIAAABfhMmioA", - "KNj6cLPRNEkqdjfOPIbg0wVhJ08TBpU_-bANzLhvKPczlle7Pq0z8QwAAAAAAAAA", + "KNj6cLPRNEkqdjfOPIbg0wVhJ08TBpU_-bANzLhvKPczlle7Pq0z8Qw", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -620,7 +620,7 @@ public static Iterable params() { new TestCase( "ip2=FE80:CD00:0000:0CDE:1257:0000:211E:729C", "XsFI2WrrLHr1O4iQAAABfhMmioA", - "KNDo3zGxO9HfN9XYJwKw2Z20h-WsBpU_-f4dSOLGSRlL1hoY2mgERuoAAAAAAAAA", + "KNDo3zGxO9HfN9XYJwKw2Z20h-WsBpU_-f4dSOLGSRlL1hoY2mgERuo", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -633,7 +633,7 @@ public static Iterable params() { new TestCase( "o.ip3=2001:db8:85a3:8d3:1319:8a2e:370:7348", "zh4dca7d-9aKOS1MAAABfhMmioA", - "KLXDcBBWJAjgJvjSdF_EJwraAQUzBpU_-ba6HZsIyKnGcbmc3KRLlmIAAAAAAAAA", + "KLXDcBBWJAjgJvjSdF_EJwraAQUzBpU_-ba6HZsIyKnGcbmc3KRLlmI", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); @@ -664,7 +664,7 @@ public static Iterable params() { new TestCase( "huge", "WZKJR_dECvXBSl3xAAABfhMmioA", - "LIe18i0rRU_Bt9vB82F46LaS9mrUkvZq1K_2Gi7UEFMhFwNXrLA_H8TLpUr4AAAAAAAAAAAAAAA", + "LIe18i0rRU_Bt9vB82F46LaS9mrUkvZq1K_2Gi7UEFMhFwNXrLA_H8TLpUr4", "2022-01-01T01:00:00.000Z", b -> { b.field("@timestamp", "2022-01-01T01:00:00Z"); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml index 14359198a8493..2c99a691f127b 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml @@ -78,11 +78,11 @@ aggretate multi_terms: - field: k8s.pod.ip - length: { aggregations.m_terms.buckets: 3 } - - match: { aggregations.m_terms.buckets.0.key_as_string: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA|10.10.55.3" } + - match: { aggregations.m_terms.buckets.0.key_as_string: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ|10.10.55.3" } - match: { aggregations.m_terms.buckets.0.doc_count: 4 } - - match: { aggregations.m_terms.buckets.1.key_as_string: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg|10.10.55.1" } + - match: { aggregations.m_terms.buckets.1.key_as_string: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o|10.10.55.1" } - match: { aggregations.m_terms.buckets.1.doc_count: 3 } - - match: { aggregations.m_terms.buckets.2.key_as_string: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg|10.10.55.2" } + - match: { aggregations.m_terms.buckets.2.key_as_string: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o|10.10.55.2" } - match: { aggregations.m_terms.buckets.2.doc_count: 1 } --- diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml index 37a5444c6d235..e6e9a8a0aa295 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml @@ -129,7 +129,7 @@ setup: - match: { hits.total.value: 9 } - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: "QxzdeAQNbMXbXqwEXmfHBbkBkHb4zgr9ViIesoONQ1ffMdWT" } + - match: { aggregations.ts.buckets.0.key: "JEMc3XgEDWzF216sBF5nxwW5AZB2-M4K_VYiHrKDjUNX3zHVkw" } - match: { aggregations.ts.buckets.0.doc_count: 9 } - gte: { aggregations.ts.buckets.0.rate.value: 0.0 } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml index 2df4187b0032a..85e1e1a5800ad 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml @@ -87,8 +87,8 @@ setup: - length: { aggregations.by_time_series.buckets: 3 } - match: aggregations.by_time_series.buckets: - "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0": - key: "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0" + "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": + key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" doc_count: 2 museum_tour: type: Feature @@ -96,8 +96,8 @@ setup: coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] type: LineString properties: { complete: true } - "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A": - key: "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A" + "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": + key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" doc_count: 5 museum_tour: type: Feature @@ -105,8 +105,8 @@ setup: coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] type: LineString properties: { complete: true } - "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH": - key: "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH" + "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": + key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" doc_count: 3 museum_tour: type: Feature @@ -140,8 +140,8 @@ setup: - length: { aggregations.by_time_series.buckets: 3 } - match: aggregations.by_time_series.buckets: - "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0": - key: "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0" + "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": + key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" doc_count: 2 museum_tour: type: Feature @@ -149,8 +149,8 @@ setup: coordinates: [ [ 2.327, 48.86 ], [ 2.336389, 48.861111 ] ] type: LineString properties: { complete: true } - "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A": - key: "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A" + "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": + key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" doc_count: 5 museum_tour: type: Feature @@ -162,8 +162,8 @@ setup: [ 4.889187, 52.373184 ] ] type: LineString properties: { complete: true } - "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH": - key: "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH" + "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": + key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" doc_count: 3 museum_tour: type: Feature @@ -216,8 +216,8 @@ setup: - length: { aggregations.by_time_series.buckets: 3 } - match: aggregations.by_time_series.buckets: - "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0": - key: "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0" + "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": + key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" doc_count: 2 museum_tour: type: Feature @@ -225,8 +225,8 @@ setup: coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] type: LineString properties: { complete: true } - "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A": - key: "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A" + "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": + key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" doc_count: 5 museum_tour: type: Feature @@ -234,8 +234,8 @@ setup: coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] type: LineString properties: { complete: true } - "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH": - key: "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH" + "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": + key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" doc_count: 3 museum_tour: type: Feature @@ -326,8 +326,8 @@ setup: - length: { aggregations.with_time_series.buckets: 3 } - match: aggregations.with_time_series.buckets: - "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0": - key: "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0" + "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": + key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" doc_count: 2 museums: doc_count_error_upper_bound: 0 @@ -342,8 +342,8 @@ setup: type: LineString properties: complete: true - "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A": - key: "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A" + "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": + key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" doc_count: 5 museums: doc_count_error_upper_bound: 0 @@ -358,8 +358,8 @@ setup: type: LineString properties: complete: true - "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH": - key: "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH" + "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": + key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" doc_count: 3 museums: doc_count_error_upper_bound: 0 @@ -447,8 +447,8 @@ setup: - length: { aggregations.with_time_series.buckets: 3 } - match: aggregations.with_time_series.buckets: - "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0": - key: "iZip5PvyDd02Wd9mTluUzhJOB4W9E7DDI8yFJM_iA1cHgel0" + "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": + key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" doc_count: 2 museums: doc_count_error_upper_bound: 0 @@ -463,8 +463,8 @@ setup: type: LineString properties: complete: true - "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A": - key: "iZip5PvyDd02Wd9mTluUzk2-mBYFTrdzTUSWqIrfkMKJUg2A" + "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": + key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" doc_count: 5 museums: doc_count_error_upper_bound: 0 @@ -488,8 +488,8 @@ setup: type: LineString properties: complete: true - "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH": - key: "iZip5PvyDd02Wd9mTluUzvcSHOWHe3vgCADk4OvknzhvOYCH" + "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": + key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" doc_count: 3 museums: doc_count_error_upper_bound: 0 From b0456abe78c0bfefb1817c0563c5469b9550b6e6 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 29 Nov 2023 14:56:03 +0100 Subject: [PATCH 067/125] fix: some more tests --- .../timeseries/TimeSeriesAggregatorTests.java | 30 +++++++++---------- .../TimeSeriesAggregatorTsidHashingTests.java | 30 +++++++++---------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index d2581118bbe69..7767ec9595950 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -66,19 +66,19 @@ public void testStandAloneTimeSeriesWithSum() throws IOException { }, ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").docCount, equalTo(4L)); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); assertThat( - ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").getAggregations().get("sum")).value(), + ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").getAggregations().get("sum")).value(), equalTo(22.0) ); - assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").docCount, equalTo(2L)); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); assertThat( - ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").getAggregations().get("sum")).value(), + ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").getAggregations().get("sum")).value(), equalTo(6.0) ); - assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").docCount, equalTo(2L)); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); assertThat( - ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").getAggregations().get("sum")).value(), + ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").getAggregations().get("sum")).value(), equalTo(8.0) ); @@ -161,24 +161,24 @@ public void testMultiBucketAggregationAsSubAggregation() throws IOException { Consumer verifier = ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").docCount, equalTo(4L)); - InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg") + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); + InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o") .getAggregations() .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 4, false, null, InternalAggregations.EMPTY)) ); - assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").docCount, equalTo(2L)); - byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA") + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); + byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg") .getAggregations() .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); - assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").docCount, equalTo(2L)); - byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg") + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); + byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4") .getAggregations() .get("by_timestamp"); assertThat( @@ -200,9 +200,9 @@ public void testAggregationSize() throws IOException { List> verifiers = new ArrayList>(); - verifiers.add(ts -> assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").docCount, equalTo(4L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").docCount, equalTo(2L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L))); for (int i = 1; i <= 3; i++) { int size = i; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java index 39785252f8708..f49ac048a1181 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java @@ -95,19 +95,19 @@ public void testStandAloneTimeSeriesWithSum() throws IOException { }, ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").docCount, equalTo(2L)); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); assertThat( - ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").getAggregations().get("sum")).value(), + ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").getAggregations().get("sum")).value(), equalTo(6.0) ); - assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").docCount, equalTo(2L)); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); assertThat( - ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").getAggregations().get("sum")).value(), + ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").getAggregations().get("sum")).value(), equalTo(8.0) ); - assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").docCount, equalTo(4L)); + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); assertThat( - ((Sum) ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").getAggregations().get("sum")).value(), + ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").getAggregations().get("sum")).value(), equalTo(22.0) ); @@ -190,24 +190,24 @@ public void testMultiBucketAggregationAsSubAggregation() throws IOException { Consumer verifier = ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").docCount, equalTo(2L)); - InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA") + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); + InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg") .getAggregations() .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); - assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").docCount, equalTo(2L)); - byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg") + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); + byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4") .getAggregations() .get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); - assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").docCount, equalTo(4L)); - byTimeStampBucket = ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg") + assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); + byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o") .getAggregations() .get("by_timestamp"); assertThat( @@ -229,9 +229,9 @@ public void testAggregationSize() throws IOException { List> verifiers = new ArrayList>(); - verifiers.add(ts -> assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMhL_RWtkZqXKUU-L12Qrh1jJj2DTLJtLqg").docCount, equalTo(4L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-Az1W934hGmwK9c1PmP91fi3PSqA").docCount, equalTo(2L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("C0uMhZuLQSpAQ5ipTZFLMmPM3J-3EIxC4a_5c4_P0nptkz5mFQadXg").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L))); for (int i = 1; i <= 3; i++) { int size = i; From b4465f1a887c7b96c4585fbe2d0f0f12c3fec68c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 29 Nov 2023 15:30:18 +0100 Subject: [PATCH 068/125] fix: downsampling and other tests --- .../rate/TimeSeriesRateAggregatorTests.java | 12 +++++++----- .../downsample/DownsampleActionSingleNodeTests.java | 1 + .../rest-api-spec/test/security/authz/70_tsdb.yml | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index 42dac7595f3c5..4b7688209b81f 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -55,11 +55,13 @@ public void testSimple() throws IOException { Consumer verifier = r -> { assertThat(r.getBuckets(), hasSize(2)); assertThat( - ((Rate) r.getBucketByKey("UVTLXgL1RxfcycWewWul_YsO9O7kwCjGCCGa8RR3lr-LdjdM").getAggregations().asList().get(0)).getValue(), + ((Rate) r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2LDvTu5MAoxgghmvEUd5a_i3Y3TA").getAggregations().asList().get(0)) + .getValue(), closeTo(59.0 / 3000.0 * MILLIS_IN_SECOND, 0.00001) ); assertThat( - ((Rate) r.getBucketByKey("UVTLXgL1RxfcycWewWul_aFGJjuWnFzk-ADiTXqzIZtkaCvS").getAggregations().asList().get(0)).getValue(), + ((Rate) r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2hRiY7lpxc5PgA4k16syGbZGgr0g").getAggregations().asList().get(0)) + .getValue(), closeTo(206.0 / 4000.0 * MILLIS_IN_SECOND, 0.00001) ); }; @@ -83,10 +85,10 @@ public void testNestedWithinDateHistogram() throws IOException { Consumer verifier = r -> { assertThat(r.getBuckets(), hasSize(2)); assertThat( - r.getBucketByKey("UVTLXgL1RxfcycWewWul_YsO9O7kwCjGCCGa8RR3lr-LdjdM"), + r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2LDvTu5MAoxgghmvEUd5a_i3Y3TA"), instanceOf(InternalTimeSeries.InternalBucket.class) ); - InternalDateHistogram hb = r.getBucketByKey("UVTLXgL1RxfcycWewWul_YsO9O7kwCjGCCGa8RR3lr-LdjdM").getAggregations().get("date"); + InternalDateHistogram hb = r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2LDvTu5MAoxgghmvEUd5a_i3Y3TA").getAggregations().get("date"); { Rate rate = hb.getBuckets().get(1).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((60 - 37 + 14) / 2000.0 * MILLIS_IN_SECOND, 0.00001)); @@ -95,7 +97,7 @@ public void testNestedWithinDateHistogram() throws IOException { Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((37 - 15) / 1000.0 * MILLIS_IN_SECOND, 0.00001)); } - hb = r.getBucketByKey("UVTLXgL1RxfcycWewWul_aFGJjuWnFzk-ADiTXqzIZtkaCvS").getAggregations().get("date"); + hb = r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2hRiY7lpxc5PgA4k16syGbZGgr0g").getAggregations().get("date"); { Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((150 - 74) / 1000.0 * MILLIS_IN_SECOND, 0.00001)); diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index c91f26243de5a..8beca42bf2a5f 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -933,6 +933,7 @@ public void testResumeDownsamplePartial() throws IOException { // NOTE: there is just one dimension with two possible values, this needs to be one of the two possible tsid values. new BytesRef( new byte[] { + 0x24, 0x42, (byte) 0xe4, (byte) 0x9f, diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml index b463a2cbaf4f7..66ce482f70603 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/security/authz/70_tsdb.yml @@ -104,7 +104,7 @@ document level security on tag: - match: {hits.total.value: 4} - length: {aggregations.tsids.buckets: 1} - - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.0.doc_count: 4} --- @@ -151,7 +151,7 @@ document level security on dimension: - match: { hits.total.value: 4 } - length: { aggregations.tsids.buckets: 1 } - - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.0.doc_count: 4} --- From c12fb39f562d0961b50a0314a075224b9a7170e6 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 30 Nov 2023 10:06:44 +0100 Subject: [PATCH 069/125] fix: a few more tests --- .../resources/rest-api-spec/test/data_stream/150_tsdb.yml | 6 +++--- .../java/org/elasticsearch/search/DocValueFormatTests.java | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index d35ec80299954..442ba800200a2 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -147,7 +147,7 @@ fetch the tsid: query: '+@timestamp:"2021-04-28T18:51:04.467Z" +k8s.pod.name:cat' - match: {hits.total.value: 1} - - match: {hits.hits.0.fields._tsid: [ "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg" ]} + - match: {hits.hits.0.fields._tsid: [ "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o" ]} --- "aggregate the tsid": @@ -168,9 +168,9 @@ fetch the tsid: _key: asc - match: {hits.total.value: 8} - - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lDmKEcP76qjItOI7LSmY_r9EAgT57T50pA"} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - - match: {aggregations.tsids.buckets.1.key: "KMQn1H8GA7xNFfZA53p2lHXagOH76qjI1lFA_LjfLfhB3LZ2mzIbSg"} + - match: {aggregations.tsids.buckets.1.key: "KCjEJ9R_BgO8TRX2QOd6dpR12oDh--qoyNZRQPy43y34Qdy2dpsyG0o"} - match: {aggregations.tsids.buckets.1.doc_count: 4} --- diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index 916637126f29f..b487f86109edd 100644 --- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -379,10 +379,12 @@ public void testParseTsid() throws IOException { timeSeriesIdBuilder.addLong("long", randomLong()); timeSeriesIdBuilder.addUnsignedLong("ulong", randomLong()); BytesRef expected = timeSeriesIdBuilder.withHash().toBytesRef(); + byte[] expectedBytes = new byte[expected.length]; + System.arraycopy(expected.bytes, 0, expectedBytes, 0, expected.length); BytesRef actual = DocValueFormat.TIME_SERIES_ID.parseBytesRef(expected); assertEquals(expected, actual); Object tsidFormat = DocValueFormat.TIME_SERIES_ID.format(expected); - Object tsidBase64 = Base64.getUrlEncoder().withoutPadding().encodeToString(expected.bytes); + Object tsidBase64 = Base64.getUrlEncoder().withoutPadding().encodeToString(expectedBytes); assertEquals(tsidFormat, tsidBase64); } } From 2a46382686290fbc450f38ce0fe5e6a494022cbe Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 30 Nov 2023 11:56:59 +0100 Subject: [PATCH 070/125] fix: some more yaml tests --- .../resources/rest-api-spec/test/reindex/100_tsdb.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml index 0b2a8a590e6c6..98996cc9c24be 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/reindex/100_tsdb.yml @@ -161,7 +161,7 @@ from tsdb to tsdb: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lL6UpbL76qjIUtiX4BxwB2rktJiZ2LipiA"} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -228,7 +228,7 @@ from standard with tsdb id to tsdb: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lL6UpbL76qjIUtiX4BxwB2rktJiZ2LipiA"} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -295,7 +295,7 @@ from standard with random _id to tsdb: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lL6UpbL76qjIUtiX4BxwB2rktJiZ2LipiA"} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-04-28T18:50:23.142Z} @@ -344,7 +344,7 @@ from tsdb to tsdb modifying timestamp: _key: asc - match: {hits.total.value: 4} - - match: {aggregations.tsids.buckets.0.key: "KMQn1H8GA7xNFfZA53p2lL6UpbL76qjIUtiX4BxwB2rktJiZ2LipiA"} + - match: {aggregations.tsids.buckets.0.key: "KCjEJ9R_BgO8TRX2QOd6dpS-lKWy--qoyFLYl-AccAdq5LSYmdi4qYg"} - match: {aggregations.tsids.buckets.0.doc_count: 4} - match: {hits.hits.0._source.@timestamp: 2021-05-28T18:50:03.142Z} - match: {hits.hits.1._source.@timestamp: 2021-05-28T18:50:23.142Z} From cb5fe2e10a1f6737e707f38c71654991bb4a35ae Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 30 Nov 2023 12:37:39 +0100 Subject: [PATCH 071/125] fix: allow warnings in yaml test --- .../rest-api-spec/test/downsample/60_settings.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml index 131aa4ce62092..74c2020d9a0d6 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml @@ -93,10 +93,13 @@ --- "Downsample datastream with tier preference": - skip: - version: " - 8.4.99" - reason: "rollup renamed to downsample in 8.5.0" + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 + features: allowed_warnings - do: + allowed_warnings: + - "index template [downsampling-template] has index patterns [test-datastream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [downsampling-template] will take precedence during new index creation" indices.put_index_template: name: downsampling-template body: From 6d54d6ae18b6b85a74d4bc6ea61a203653155b89 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 30 Nov 2023 13:35:25 +0100 Subject: [PATCH 072/125] Bring back the property of the time_series agg to use tsid is bucket key instead of tsid hash. --- .../metrics/geoline-aggregation.asciidoc | 26 +- .../bucket/TimeSeriesAggregationsIT.java | 227 ++++++++++++++ ...riesAggregationsUnlimitedDimensionsIT.java | 3 +- .../TimeSeriesNestedAggregationsIT.java | 3 +- .../bucket/timeseries/InternalTimeSeries.java | 8 +- .../bucket/timeseries/ParsedTimeSeries.java | 5 +- .../timeseries/TimeSeriesAggregator.java | 61 +++- .../timeseries/InternalTimeSeriesTests.java | 7 +- .../InternalTimeSeriesTsidHashingTests.java | 180 ----------- .../timeseries/TimeSeriesAggregatorTests.java | 104 ++++--- .../TimeSeriesAggregatorTsidHashingTests.java | 281 ------------------ .../test/aggregations/time_series.yml | 24 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 38 +++ .../index/query/SearchExecutionContext.java | 14 + .../AggregationExecutionContext.java | 4 +- .../support/TimeSeriesIndexSearcherTests.java | 8 +- .../aggregations/AggregatorTestCase.java | 9 +- .../rate/TimeSeriesRateAggregator.java | 4 +- .../downsample/DownsampleShardIndexer.java | 18 +- 19 files changed, 460 insertions(+), 564 deletions(-) delete mode 100644 modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTsidHashingTests.java delete mode 100644 modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java diff --git a/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc b/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc index 4d90ec75a3dc3..aabe8d172e4a0 100644 --- a/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/geoline-aggregation.asciidoc @@ -347,8 +347,10 @@ This query will result in: "aggregations": { "path": { "buckets": { - "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": { - "key": "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA", + "{city=Paris}": { + "key": { + "city": "Paris" + }, "doc_count": 2, "museum_tour": { "type": "Feature", @@ -361,13 +363,15 @@ This query will result in: } } }, - "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": { - "key": "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA", - "doc_count": 5, + "{city=Antwerp}": { + "key": { + "city": "Antwerp" + }, + "doc_count": 3, "museum_tour": { "type": "Feature", "geometry": { - "coordinates": [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ], + "coordinates": [ [ 4.401384, 51.220292 ], [ 4.405819, 51.221758 ], [ 4.4052, 51.2229 ] ], "type": "LineString" }, "properties": { @@ -375,13 +379,15 @@ This query will result in: } } }, - "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": { - "key": "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw", - "doc_count": 3, + "{city=Amsterdam}": { + "key": { + "city": "Amsterdam" + }, + "doc_count": 5, "museum_tour": { "type": "Feature", "geometry": { - "coordinates": [ [ 4.401384, 51.220292 ], [ 4.405819, 51.221758 ], [ 4.4052, 51.2229 ] ], + "coordinates": [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ], "type": "LineString" }, "properties": { diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java index cbd06b1b70020..2050ce20b1aee 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java @@ -15,28 +15,55 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.aggregations.AggregationIntegTestCase; +import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries; import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.PipelineAggregatorBuilders; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.elasticsearch.search.aggregations.bucket.global.Global; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.metrics.CompensatedSum; +import org.elasticsearch.search.aggregations.metrics.Stats; +import org.elasticsearch.search.aggregations.metrics.Sum; +import org.elasticsearch.search.aggregations.pipeline.SimpleValue; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.elasticsearch.search.aggregations.AggregationBuilders.dateHistogram; +import static org.elasticsearch.search.aggregations.AggregationBuilders.global; +import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; +import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; +import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.search.aggregations.AggregationBuilders.topHits; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; @ESIntegTestCase.SuiteScopeTestCase public class TimeSeriesAggregationsIT extends AggregationIntegTestCase { @@ -148,6 +175,206 @@ public void setupSuiteScopeCluster() throws Exception { indexRandom(true, false, docs); } + public void testStandAloneTimeSeriesAgg() { + assertNoFailuresAndResponse(prepareSearch("index").setSize(0).addAggregation(timeSeries("by_ts")), response -> { + Aggregations aggregations = response.getAggregations(); + assertNotNull(aggregations); + InternalTimeSeries timeSeries = aggregations.get("by_ts"); + assertThat( + timeSeries.getBuckets().stream().map(MultiBucketsAggregation.Bucket::getKey).collect(Collectors.toSet()), + equalTo(data.keySet()) + ); + for (InternalTimeSeries.Bucket bucket : timeSeries.getBuckets()) { + @SuppressWarnings("unchecked") + Map key = (Map) bucket.getKey(); + assertThat((long) data.get(key).size(), equalTo(bucket.getDocCount())); + } + }); + } + + public void testTimeSeriesGroupedByADimension() { + String groupBy = "dim_" + randomIntBetween(0, numberOfDimensions - 1); + assertNoFailuresAndResponse( + prepareSearch("index").setSize(0) + .addAggregation( + terms("by_dim").field(groupBy) + .size(data.size()) + .collectMode(randomFrom(Aggregator.SubAggCollectionMode.values())) + .subAggregation(timeSeries("by_ts")) + ), + response -> { + Aggregations aggregations = response.getAggregations(); + assertNotNull(aggregations); + Terms terms = aggregations.get("by_dim"); + Set> keys = new HashSet<>(); + for (Terms.Bucket term : terms.getBuckets()) { + InternalTimeSeries timeSeries = term.getAggregations().get("by_ts"); + for (InternalTimeSeries.Bucket bucket : timeSeries.getBuckets()) { + @SuppressWarnings("unchecked") + Map key = (Map) bucket.getKey(); + assertThat((long) data.get(key).size(), equalTo(bucket.getDocCount())); + assertTrue("key is not unique", keys.add(key)); + assertThat( + "time series doesn't contain dimensions we grouped by", + key.get(groupBy), + equalTo(term.getKeyAsString()) + ); + } + } + assertThat(keys, equalTo(data.keySet())); + } + ); + } + + public void testTimeSeriesGroupedByDateHistogram() { + DateHistogramInterval fixedInterval = DateHistogramInterval.days(randomIntBetween(10, 100)); + assertNoFailuresAndResponse( + prepareSearch("index").setSize(0) + .addAggregation( + dateHistogram("by_time").field("@timestamp") + .fixedInterval(fixedInterval) + .subAggregation(timeSeries("by_ts").subAggregation(stats("timestamp").field("@timestamp"))) + ), + response -> { + Aggregations aggregations = response.getAggregations(); + assertNotNull(aggregations); + Histogram histogram = aggregations.get("by_time"); + Map, Long> keys = new HashMap<>(); + for (Histogram.Bucket interval : histogram.getBuckets()) { + long intervalStart = ((ZonedDateTime) interval.getKey()).toEpochSecond() * 1000; + long intervalEnd = intervalStart + fixedInterval.estimateMillis(); + InternalTimeSeries timeSeries = interval.getAggregations().get("by_ts"); + for (InternalTimeSeries.Bucket bucket : timeSeries.getBuckets()) { + @SuppressWarnings("unchecked") + Map key = (Map) bucket.getKey(); + keys.compute(key, (k, v) -> (v == null ? 0 : v) + bucket.getDocCount()); + assertThat(bucket.getDocCount(), lessThanOrEqualTo((long) data.get(key).size())); + Stats stats = bucket.getAggregations().get("timestamp"); + long minTimestamp = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(stats.getMinAsString()); + long maxTimestamp = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(stats.getMaxAsString()); + assertThat(minTimestamp, greaterThanOrEqualTo(intervalStart)); + assertThat(maxTimestamp, lessThan(intervalEnd)); + } + } + assertThat(keys.keySet(), equalTo(data.keySet())); + for (Map.Entry, Long> entry : keys.entrySet()) { + assertThat(entry.getValue(), equalTo((long) data.get(entry.getKey()).size())); + } + } + ); + } + + public void testStandAloneTimeSeriesAggWithDimFilter() { + boolean include = randomBoolean(); + int dim = randomIntBetween(0, numberOfDimensions - 1); + String val = randomFrom(dimensions[dim]); + QueryBuilder queryBuilder = QueryBuilders.termQuery("dim_" + dim, val); + if (include == false) { + queryBuilder = QueryBuilders.boolQuery().mustNot(queryBuilder); + } + assertNoFailuresAndResponse( + prepareSearch("index").setQuery(queryBuilder).setSize(0).addAggregation(timeSeries("by_ts")), + response -> { + Aggregations aggregations = response.getAggregations(); + assertNotNull(aggregations); + InternalTimeSeries timeSeries = aggregations.get("by_ts"); + Map, Map>> filteredData = dataFilteredByDimension("dim_" + dim, val, include); + assertThat( + timeSeries.getBuckets().stream().map(MultiBucketsAggregation.Bucket::getKey).collect(Collectors.toSet()), + equalTo(filteredData.keySet()) + ); + for (InternalTimeSeries.Bucket bucket : timeSeries.getBuckets()) { + @SuppressWarnings("unchecked") + Map key = (Map) bucket.getKey(); + assertThat(bucket.getDocCount(), equalTo((long) filteredData.get(key).size())); + } + } + ); + } + + public void testStandAloneTimeSeriesAggWithGlobalAggregation() { + boolean include = randomBoolean(); + int dim = randomIntBetween(0, numberOfDimensions - 1); + int metric = randomIntBetween(0, numberOfMetrics - 1); + String val = randomFrom(dimensions[dim]); + QueryBuilder queryBuilder = QueryBuilders.termQuery("dim_" + dim, val); + if (include == false) { + queryBuilder = QueryBuilders.boolQuery().mustNot(queryBuilder); + } + assertNoFailuresAndResponse( + prepareSearch("index").setQuery(queryBuilder) + .setSize(0) + .addAggregation(timeSeries("by_ts").subAggregation(sum("filter_sum").field("metric_" + metric))) + .addAggregation(global("everything").subAggregation(sum("all_sum").field("metric_" + metric))) + .addAggregation(PipelineAggregatorBuilders.sumBucket("total_filter_sum", "by_ts>filter_sum")), + response -> { + Aggregations aggregations = response.getAggregations(); + assertNotNull(aggregations); + InternalTimeSeries timeSeries = aggregations.get("by_ts"); + Map, Map>> filteredData = dataFilteredByDimension("dim_" + dim, val, include); + assertThat( + timeSeries.getBuckets().stream().map(MultiBucketsAggregation.Bucket::getKey).collect(Collectors.toSet()), + equalTo(filteredData.keySet()) + ); + for (InternalTimeSeries.Bucket bucket : timeSeries.getBuckets()) { + @SuppressWarnings("unchecked") + Map key = (Map) bucket.getKey(); + assertThat(bucket.getDocCount(), equalTo((long) filteredData.get(key).size())); + } + SimpleValue obj = aggregations.get("total_filter_sum"); + assertThat(obj.value(), closeTo(sumByMetric(filteredData, "metric_" + metric), obj.value() * 0.0001)); + + Global global = aggregations.get("everything"); + Sum allSum = global.getAggregations().get("all_sum"); + assertThat(allSum.value(), closeTo(sumByMetric(data, "metric_" + metric), allSum.value() * 0.0001)); + } + ); + + ElasticsearchException e = expectThrows( + ElasticsearchException.class, + () -> prepareSearch("index").setQuery(QueryBuilders.termQuery("dim_" + dim, val)) + .setSize(0) + .addAggregation(global("everything").subAggregation(timeSeries("by_ts"))) + .get() + ); + assertThat(e.getRootCause().getMessage(), containsString("Time series aggregations cannot be used inside global aggregation.")); + } + + public void testStandAloneTimeSeriesAggWithMetricFilter() { + boolean above = randomBoolean(); + int metric = randomIntBetween(0, numberOfMetrics - 1); + double val = randomDoubleBetween(0, 100000, true); + RangeQueryBuilder queryBuilder = QueryBuilders.rangeQuery("metric_" + metric); + if (above) { + queryBuilder.gt(val); + } else { + queryBuilder.lte(val); + } + assertNoFailuresAndResponse( + prepareSearch("index").setQuery(queryBuilder).setSize(0).addAggregation(timeSeries("by_ts")), + response -> { + Aggregations aggregations = response.getAggregations(); + assertNotNull(aggregations); + InternalTimeSeries timeSeries = aggregations.get("by_ts"); + Map, Map>> filteredData = dataFilteredByMetric( + data, + "metric_" + metric, + val, + above + ); + assertThat( + timeSeries.getBuckets().stream().map(MultiBucketsAggregation.Bucket::getKey).collect(Collectors.toSet()), + equalTo(filteredData.keySet()) + ); + for (InternalTimeSeries.Bucket bucket : timeSeries.getBuckets()) { + @SuppressWarnings("unchecked") + Map key = (Map) bucket.getKey(); + assertThat(bucket.getDocCount(), equalTo((long) filteredData.get(key).size())); + } + } + ); + } + public void testRetrievingHits() { Map.Entry filterMetric = randomMetricAndValue(data); double lowerVal = filterMetric.getValue() - randomDoubleBetween(0, 100000, true); diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java index 7793a687a4e74..c6d9a692e92c0 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.util.Iterator; -import java.util.List; import java.util.Set; import java.util.TreeSet; import java.util.function.Supplier; @@ -194,7 +193,7 @@ public void testCardinalityByTsid() { } private static void assertTimeSeriesAggregation(final InternalTimeSeries timeSeriesAggregation) { - final List dimensions = timeSeriesAggregation.getBuckets().stream().map(InternalTimeSeries.InternalBucket::getKey).toList(); + final var dimensions = timeSeriesAggregation.getBuckets().stream().map(InternalTimeSeries.InternalBucket::getKey).toList(); // NOTE: only two time series expected as a result of having just two distinct values for the last dimension assertEquals(2, dimensions.size()); diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java index 057cf2c82f836..a2ff9bd85f38b 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java @@ -33,7 +33,6 @@ import java.io.IOException; import java.util.Iterator; -import java.util.List; import java.util.Locale; import java.util.Set; import java.util.TreeSet; @@ -207,7 +206,7 @@ public void testCardinalityByTsid() { } private static void assertTimeSeriesAggregation(final InternalTimeSeries timeSeriesAggregation) { - final List dimensions = timeSeriesAggregation.getBuckets().stream().map(InternalTimeSeries.InternalBucket::getKey).toList(); + final var dimensions = timeSeriesAggregation.getBuckets().stream().map(InternalTimeSeries.InternalBucket::getKey).toList(); // NOTE: only two time series expected as a result of having just two distinct values for the last dimension assertEquals(2, dimensions.size()); diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java index 8604f83f6a2b6..9b80002416c6b 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java @@ -81,8 +81,8 @@ public void writeTo(StreamOutput out) throws IOException { } @Override - public Object getKey() { - return TimeSeriesIdFieldMapper.decodeTsid(key); + public Map getKey() { + return TimeSeriesIdFieldMapper.decodeTsid2(key); } @Override @@ -221,7 +221,6 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurr ? ((TimeSeriesAggregationBuilder) reduceContext.builder()).getSize() : null; // tests may use a fake builder List bucketsWithSameKey = new ArrayList<>(aggregations.size()); - BytesRef prevTsid = null; while (pq.size() > 0) { reduceContext.consumeBucketsAndMaybeBreak(1); bucketsWithSameKey.clear(); @@ -247,13 +246,10 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurr } else { reducedBucket = reduceBucket(bucketsWithSameKey, reduceContext); } - BytesRef tsid = reducedBucket.key; - assert prevTsid == null || tsid.compareTo(prevTsid) > 0; reduced.buckets.add(reducedBucket); if (size != null && reduced.buckets.size() >= size) { break; } - prevTsid = tsid; } return reduced; } diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/ParsedTimeSeries.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/ParsedTimeSeries.java index 9a45c4554137c..f4eff3e878b59 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/ParsedTimeSeries.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/ParsedTimeSeries.java @@ -16,6 +16,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; public class ParsedTimeSeries extends ParsedMultiBucketAggregation { @@ -62,7 +63,7 @@ public static ParsedTimeSeries fromXContent(XContentParser parser, String name) static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket { - private Object key; + private Map key; @Override public Object getKey() { @@ -75,7 +76,7 @@ public String getKeyAsString() { } static ParsedTimeSeries.ParsedBucket fromXContent(XContentParser parser, boolean keyed) throws IOException { - return parseXContent(parser, keyed, ParsedTimeSeries.ParsedBucket::new, (p, bucket) -> bucket.key = p.text()); + return parseXContent(parser, keyed, ParsedTimeSeries.ParsedBucket::new, (p, bucket) -> bucket.key = new TreeMap<>(p.map())); } } diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java index d5b796f3b93c5..da4cf426c5250 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java @@ -8,8 +8,11 @@ package org.elasticsearch.aggregations.bucket.timeseries; +import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.core.Releasables; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.search.aggregations.AggregationExecutionContext; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -20,11 +23,15 @@ import org.elasticsearch.search.aggregations.bucket.BucketsAggregator; import org.elasticsearch.search.aggregations.bucket.terms.BytesKeyedBucketOrds; import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; public class TimeSeriesAggregator extends BucketsAggregator { @@ -32,6 +39,8 @@ public class TimeSeriesAggregator extends BucketsAggregator { private final boolean keyed; private final int size; + private final SortedMap dimensionValueSources; + @SuppressWarnings("this-escape") public TimeSeriesAggregator( String name, @@ -47,6 +56,11 @@ public TimeSeriesAggregator( this.keyed = keyed; bucketOrds = BytesKeyedBucketOrds.build(bigArrays(), bucketCardinality); this.size = size; + dimensionValueSources = new TreeMap<>(); + for (var dim : context.subSearchContext().getSearchExecutionContext().dimensionFields()) { + var valueSource = ValuesSourceConfig.resolve(context, null, dim.name(), null, null, null, null, null).getValuesSource(); + dimensionValueSources.put(dim.name(), valueSource); + } } @Override @@ -56,14 +70,11 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I for (int ordIdx = 0; ordIdx < owningBucketOrds.length; ordIdx++) { BytesKeyedBucketOrds.BucketOrdsEnum ordsEnum = bucketOrds.ordsEnum(owningBucketOrds[ordIdx]); List buckets = new ArrayList<>(); - BytesRef prev = null; while (ordsEnum.next()) { long docCount = bucketDocCount(ordsEnum.ord()); ordsEnum.readValue(spare); - assert prev == null || spare.compareTo(prev) > 0 - : "key [" + spare.utf8ToString() + "] is smaller than previous key [" + prev.utf8ToString() + "]"; InternalTimeSeries.InternalBucket bucket = new InternalTimeSeries.InternalBucket( - prev = BytesRef.deepCopyOf(spare), // Closing bucketOrds will corrupt the bytes ref, so need to make a deep copy here. + BytesRef.deepCopyOf(spare), // Closing bucketOrds will corrupt the bytes ref, so need to make a deep copy here. docCount, null, keyed @@ -95,8 +106,36 @@ protected void doClose() { Releasables.close(bucketOrds); } + @FunctionalInterface + interface TsidConsumer { + void accept(int docId, TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder) throws IOException; + } + @Override protected LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, LeafBucketCollector sub) throws IOException { + SortedMap map = new TreeMap<>(); + for (var entry : dimensionValueSources.entrySet()) { + String fieldName = entry.getKey(); + if (entry.getValue() instanceof ValuesSource.Numeric numericVS) { + SortedNumericDocValues docValues = numericVS.longValues(aggCtx.getLeafReaderContext()); + map.put(entry.getKey(), (docId, tsidBuilder) -> { + if (docValues.advanceExact(docId)) { + for (int i = 0; i < docValues.docValueCount(); i++) { + tsidBuilder.addLong(fieldName, docValues.nextValue()); + } + } + }); + } else { + SortedBinaryDocValues docValues = entry.getValue().bytesValues(aggCtx.getLeafReaderContext()); + map.put(entry.getKey(), (docId, tsidBuilder) -> { + if (docValues.advanceExact(docId)) { + for (int i = 0; i < docValues.docValueCount(); i++) { + tsidBuilder.addString(fieldName, docValues.nextValue()); + } + } + }); + } + } return new LeafBucketCollectorBase(sub, null) { // Keeping track of these fields helps to reduce time spent attempting to add bucket + tsid combos that already were added. @@ -111,12 +150,19 @@ public void collect(int doc, long bucket) throws IOException { // changes to what is stored in currentTsidOrd then that ordinal well never occur again. Same applies // currentBucket if there is no parent aggregation or the immediate parent aggregation creates buckets // based on @timestamp field or dimension fields (fields that make up the tsid). - if (currentBucket == bucket && currentTsidOrd == aggCtx.getTsidOrd()) { + if (currentBucket == bucket && currentTsidOrd == aggCtx.getTsidHashOrd()) { collectExistingBucket(sub, doc, currentBucketOrdinal); return; } - long bucketOrdinal = bucketOrds.add(bucket, aggCtx.getTsid()); + TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); + for (TsidConsumer consumer : map.values()) { + consumer.accept(doc, tsidBuilder); + } + + BytesRef tsid = tsidBuilder.withoutHash().toBytesRef(); + // TimeSeriesIdFieldMapper.decodeTsid2(tsid); + long bucketOrdinal = bucketOrds.add(bucket, tsid); if (bucketOrdinal < 0) { // already seen bucketOrdinal = -1 - bucketOrdinal; collectExistingBucket(sub, doc, bucketOrdinal); @@ -125,9 +171,10 @@ public void collect(int doc, long bucket) throws IOException { } currentBucketOrdinal = bucketOrdinal; - currentTsidOrd = aggCtx.getTsidOrd(); + currentTsidOrd = aggCtx.getTsidHashOrd(); currentBucket = bucket; } + }; } diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java index cc65968ec77ed..e7255ecbff395 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java @@ -53,7 +53,7 @@ private List randomBuckets(boolean keyed, InternalAggregations a builder.addString(entry.getKey(), (String) entry.getValue()); } try { - var key = builder.withHash().toBytesRef(); + var key = builder.withoutHash().toBytesRef(); bucketList.add(new InternalBucket(key, docCount, aggregations, keyed)); } catch (IOException e) { throw new UncheckedIOException(e); @@ -89,12 +89,11 @@ protected InternalTimeSeries createTestInstance(String name, Map } @Override - @SuppressWarnings("unchecked") protected void assertReduced(InternalTimeSeries reduced, List inputs) { - Map keys = new HashMap<>(); + Map, Long> keys = new HashMap<>(); for (InternalTimeSeries in : inputs) { for (InternalBucket bucket : in.getBuckets()) { - keys.compute((String) bucket.getKey(), (k, v) -> { + keys.compute(bucket.getKey(), (k, v) -> { if (v == null) { return bucket.docCount; } else { diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTsidHashingTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTsidHashingTests.java deleted file mode 100644 index 1a68ce11f71eb..0000000000000 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTsidHashingTests.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.aggregations.bucket.timeseries; - -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.aggregations.bucket.AggregationMultiBucketAggregationTestCase; -import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries.InternalBucket; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.MockBigArrays; -import org.elasticsearch.common.util.MockPageCacheRecycler; -import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; -import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.AggregationReduceContext; -import org.elasticsearch.search.aggregations.InternalAggregations; -import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.elasticsearch.xcontent.ContextParser; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.function.Predicate; - -import static org.hamcrest.Matchers.arrayContainingInAnyOrder; -import static org.hamcrest.Matchers.equalTo; - -public class InternalTimeSeriesTsidHashingTests extends AggregationMultiBucketAggregationTestCase { - - @Override - protected Map.Entry> getParser() { - return Map.entry(TimeSeriesAggregationBuilder.NAME, (p, c) -> ParsedTimeSeries.fromXContent(p, (String) c)); - } - - private List randomBuckets(boolean keyed, InternalAggregations aggregations) { - int numberOfBuckets = randomNumberOfBuckets(); - List bucketList = new ArrayList<>(numberOfBuckets); - List> keys = randomKeys(bucketKeys(randomIntBetween(1, 4)), numberOfBuckets); - for (int j = 0; j < numberOfBuckets; j++) { - long docCount = randomLongBetween(0, Long.MAX_VALUE / (20L * numberOfBuckets)); - var builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); - for (var entry : keys.get(j).entrySet()) { - builder.addString(entry.getKey(), (String) entry.getValue()); - } - try { - var key = builder.withHash().toBytesRef(); - bucketList.add(new InternalBucket(key, docCount, aggregations, keyed)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - // The interal time series' reduce method expects for each shard level response that the buckets are sorted by tsid: - bucketList.sort(Comparator.comparing(o -> o.key)); - return bucketList; - } - - private List bucketKeys(int numberOfKeys) { - return randomUnique(() -> randomAlphaOfLength(10), numberOfKeys).stream().toList(); - } - - private List> randomKeys(List bucketKeys, int numberOfBuckets) { - List> keys = new ArrayList<>(); - for (int i = 0; i < numberOfBuckets; i++) { - keys.add(randomValueOtherThanMany(keys::contains, () -> { - Map key = new TreeMap<>(); - for (String name : bucketKeys) { - key.put(name, randomAlphaOfLength(4)); - } - return key; - })); - } - return keys; - } - - @Override - protected InternalTimeSeries createTestInstance(String name, Map metadata, InternalAggregations aggregations) { - boolean keyed = randomBoolean(); - return new InternalTimeSeries(name, randomBuckets(keyed, aggregations), keyed, metadata); - } - - @Override - @SuppressWarnings("unchecked") - protected void assertReduced(InternalTimeSeries reduced, List inputs) { - Map keys = new HashMap<>(); - for (InternalTimeSeries in : inputs) { - for (InternalBucket bucket : in.getBuckets()) { - keys.compute((String) bucket.getKey(), (k, v) -> { - if (v == null) { - return bucket.docCount; - } else { - return bucket.docCount + v; - } - }); - } - } - assertThat( - reduced.getBuckets().stream().map(InternalBucket::getKey).toArray(Object[]::new), - arrayContainingInAnyOrder(keys.keySet().toArray(Object[]::new)) - ); - } - - @Override - protected Class implementationClass() { - return ParsedTimeSeries.class; - } - - @Override - protected Predicate excludePathsFromXContentInsertion() { - return s -> s.endsWith(".key"); - } - - public void testReduceSimple() { - // a simple test, to easily spot easy mistakes in the merge logic in InternalTimeSeries#reduce(...) method. - InternalTimeSeries first = new InternalTimeSeries( - "ts", - List.of( - new InternalBucket(new BytesRef("1"), 3, InternalAggregations.EMPTY, false), - new InternalBucket(new BytesRef("10"), 6, InternalAggregations.EMPTY, false), - new InternalBucket(new BytesRef("2"), 2, InternalAggregations.EMPTY, false), - new InternalBucket(new BytesRef("9"), 5, InternalAggregations.EMPTY, false) - ), - false, - Map.of() - ); - InternalTimeSeries second = new InternalTimeSeries( - "ts", - List.of( - new InternalBucket(new BytesRef("2"), 1, InternalAggregations.EMPTY, false), - new InternalBucket(new BytesRef("3"), 3, InternalAggregations.EMPTY, false) - ), - false, - Map.of() - ); - InternalTimeSeries third = new InternalTimeSeries( - "ts", - List.of( - new InternalBucket(new BytesRef("1"), 2, InternalAggregations.EMPTY, false), - new InternalBucket(new BytesRef("3"), 4, InternalAggregations.EMPTY, false), - new InternalBucket(new BytesRef("9"), 4, InternalAggregations.EMPTY, false) - ), - false, - Map.of() - ); - AggregationReduceContext context = new AggregationReduceContext.ForFinal( - new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()), - mockScriptService(), - () -> false, - new TimeSeriesAggregationBuilder("ts"), - value -> {}, - PipelineAggregator.PipelineTree.EMPTY - ); - - InternalTimeSeries result = (InternalTimeSeries) first.reduce(List.of(first, second, third), context); - assertThat(result.getBuckets().get(0).key.utf8ToString(), equalTo("1")); - assertThat(result.getBuckets().get(0).getDocCount(), equalTo(5L)); - assertThat(result.getBuckets().get(1).key.utf8ToString(), equalTo("10")); - assertThat(result.getBuckets().get(1).getDocCount(), equalTo(6L)); - assertThat(result.getBuckets().get(2).key.utf8ToString(), equalTo("2")); - assertThat(result.getBuckets().get(2).getDocCount(), equalTo(3L)); - assertThat(result.getBuckets().get(3).key.utf8ToString(), equalTo("3")); - assertThat(result.getBuckets().get(3).getDocCount(), equalTo(7L)); - assertThat(result.getBuckets().get(4).key.utf8ToString(), equalTo("9")); - assertThat(result.getBuckets().get(4).getDocCount(), equalTo(9L)); - } - - @Override - protected InternalTimeSeries mutateInstance(InternalTimeSeries instance) { - return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929 - } -} diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index 7767ec9595950..2618cb83c7ff8 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -14,16 +14,20 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.aggregations.bucket.AggregationTestCase; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper.TimeSeriesIdBuilder; @@ -66,25 +70,20 @@ public void testStandAloneTimeSeriesWithSum() throws IOException { }, ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); - assertThat( - ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").getAggregations().get("sum")).value(), - equalTo(22.0) - ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); - assertThat( - ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").getAggregations().get("sum")).value(), - equalTo(6.0) - ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); - assertThat( - ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").getAggregations().get("sum")).value(), - equalTo(8.0) - ); + assertThat(ts.getBucketByKey("{dim1=aaa, dim2=xxx}").docCount, equalTo(2L)); + assertThat(((Sum) ts.getBucketByKey("{dim1=aaa, dim2=xxx}").getAggregations().get("sum")).value(), equalTo(6.0)); + assertThat(ts.getBucketByKey("{dim1=aaa, dim2=yyy}").docCount, equalTo(2L)); + assertThat(((Sum) ts.getBucketByKey("{dim1=aaa, dim2=yyy}").getAggregations().get("sum")).value(), equalTo(8.0)); + assertThat(ts.getBucketByKey("{dim1=bbb, dim2=zzz}").docCount, equalTo(4L)); + assertThat(((Sum) ts.getBucketByKey("{dim1=bbb, dim2=zzz}").getAggregations().get("sum")).value(), equalTo(22.0)); }, - new KeywordFieldMapper.KeywordFieldType("dim1"), - new KeywordFieldMapper.KeywordFieldType("dim2"), + new KeywordFieldMapper.Builder("dim1", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType(), + new KeywordFieldMapper.Builder("dim2", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType(), new NumberFieldMapper.NumberFieldType("val1", NumberFieldMapper.NumberType.INTEGER) ); } @@ -97,8 +96,16 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens for (int i = 0; i < dimensions.length; i += 2) { if (dimensions[i + 1] instanceof Number n) { builder.addLong(dimensions[i].toString(), n.longValue()); + if (dimensions[i + 1] instanceof Integer || dimensions[i + 1] instanceof Long) { + fields.add(new NumericDocValuesField(dimensions[i].toString(), ((Number) dimensions[i + 1]).longValue())); + } else if (dimensions[i + 1] instanceof Float) { + fields.add(new FloatDocValuesField(dimensions[i].toString(), (float) dimensions[i + 1])); + } else if (dimensions[i + 1] instanceof Double) { + fields.add(new DoubleDocValuesField(dimensions[i].toString(), (double) dimensions[i + 1])); + } } else { builder.addString(dimensions[i].toString(), dimensions[i + 1].toString()); + fields.add(new SortedSetDocValuesField(dimensions[i].toString(), new BytesRef(dimensions[i + 1].toString()))); } } for (int i = 0; i < metrics.length; i += 2) { @@ -110,8 +117,7 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens fields.add(new DoubleDocValuesField(metrics[i].toString(), (double) metrics[i + 1])); } } - fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.withHash().toBytesRef())); - // TODO: Handle metrics + fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.withoutHash().toBytesRef())); iw.addDocument(fields); } @@ -140,7 +146,9 @@ public void testWithDateHistogramExecutedAsFilterByFilterWithTimeSeriesIndexSear aggregationBuilder, TimeSeriesIdFieldMapper.FIELD_TYPE, new DateFieldMapper.DateFieldType("@timestamp"), - new KeywordFieldMapper.KeywordFieldType("dim1"), + new KeywordFieldMapper.Builder("dim1", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType(), new NumberFieldMapper.NumberFieldType("val1", NumberFieldMapper.NumberType.INTEGER) ).withQuery(new MatchAllDocsQuery()) ); @@ -161,29 +169,23 @@ public void testMultiBucketAggregationAsSubAggregation() throws IOException { Consumer verifier = ts -> { assertThat(ts.getBuckets(), hasSize(3)); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); - InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o") - .getAggregations() - .get("by_timestamp"); + assertThat(ts.getBucketByKey("{dim1=aaa, dim2=xxx}").docCount, equalTo(2L)); + InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("{dim1=aaa, dim2=xxx}").getAggregations().get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), - contains(new InternalDateHistogram.Bucket(startTime, 4, false, null, InternalAggregations.EMPTY)) + contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); - byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg") - .getAggregations() - .get("by_timestamp"); + assertThat(ts.getBucketByKey("{dim1=aaa, dim2=yyy}").docCount, equalTo(2L)); + byTimeStampBucket = ts.getBucketByKey("{dim1=aaa, dim2=yyy}").getAggregations().get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); - byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4") - .getAggregations() - .get("by_timestamp"); + assertThat(ts.getBucketByKey("{dim1=bbb, dim2=zzz}").docCount, equalTo(4L)); + byTimeStampBucket = ts.getBucketByKey("{dim1=bbb, dim2=zzz}").getAggregations().get("by_timestamp"); assertThat( byTimeStampBucket.getBuckets(), - contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) + contains(new InternalDateHistogram.Bucket(startTime, 4, false, null, InternalAggregations.EMPTY)) ); }; @@ -192,7 +194,18 @@ public void testMultiBucketAggregationAsSubAggregation() throws IOException { dateBuilder.fixedInterval(DateHistogramInterval.seconds(1)); TimeSeriesAggregationBuilder tsBuilder = new TimeSeriesAggregationBuilder("by_tsid"); tsBuilder.subAggregation(dateBuilder); - timeSeriesTestCase(tsBuilder, new MatchAllDocsQuery(), buildIndex, verifier); + timeSeriesTestCase( + tsBuilder, + new MatchAllDocsQuery(), + buildIndex, + verifier, + new KeywordFieldMapper.Builder("dim1", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType(), + new KeywordFieldMapper.Builder("dim2", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType() + ); } public void testAggregationSize() throws IOException { @@ -200,11 +213,11 @@ public void testAggregationSize() throws IOException { List> verifiers = new ArrayList>(); - verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=aaa, dim2=xxx}").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=aaa, dim2=yyy}").docCount, equalTo(2L))); + verifiers.add(ts -> assertThat(ts.getBucketByKey("{dim1=bbb, dim2=zzz}").docCount, equalTo(2L))); - for (int i = 1; i <= 3; i++) { + for (int i = 1; i < 3; i++) { int size = i; Consumer limitedVerifier = ts -> { assertThat(ts.getBuckets(), hasSize(size)); @@ -216,7 +229,18 @@ public void testAggregationSize() throws IOException { TimeSeriesAggregationBuilder limitedTsBuilder = new TimeSeriesAggregationBuilder("by_tsid"); limitedTsBuilder.setSize(i); - timeSeriesTestCase(limitedTsBuilder, new MatchAllDocsQuery(), buildIndex, limitedVerifier); + timeSeriesTestCase( + limitedTsBuilder, + new MatchAllDocsQuery(), + buildIndex, + limitedVerifier, + new KeywordFieldMapper.Builder("dim1", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType(), + new KeywordFieldMapper.Builder("dim2", IndexVersion.current()).dimension(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType() + ); } } diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java deleted file mode 100644 index f49ac048a1181..0000000000000 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTsidHashingTests.java +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.aggregations.bucket.timeseries; - -import org.apache.lucene.document.DoubleDocValuesField; -import org.apache.lucene.document.FloatDocValuesField; -import org.apache.lucene.document.LongPoint; -import org.apache.lucene.document.NumericDocValuesField; -import org.apache.lucene.document.SortedDocValuesField; -import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.tests.index.RandomIndexWriter; -import org.elasticsearch.aggregations.bucket.AggregationTestCase; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; -import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; -import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.KeywordFieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.NumberFieldMapper; -import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; -import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper.TimeSeriesIdBuilder; -import org.elasticsearch.search.aggregations.InternalAggregations; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.histogram.InternalDateHistogram; -import org.elasticsearch.search.aggregations.metrics.Sum; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.test.index.IndexVersionUtils; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Consumer; - -import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; - -public class TimeSeriesAggregatorTsidHashingTests extends AggregationTestCase { - - @Override - protected List getSupportedValuesSourceTypes() { - return List.of(); - } - - /** - * @return {@link IndexSettings} including a specific {@link IndexVersion}. Versions up to 8.11.0 do not use tsid hashing and - * {@link InternalTimeSeries#getBucketByKey(String)} works passing dimension name/value pairs. After introducing tsis hashing - * that is not possible anymore and we need to get buckets by the tsid value instead. - */ - @Override - protected IndexSettings createIndexSettings() { - final IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.V_8_10_0, // TODO: change to 8.11.0 after release - IndexVersion.current() - ); - return new IndexSettings( - IndexMetadata.builder("_index") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, indexVersion)) - .numberOfShards(1) - .numberOfReplicas(0) - .creationDate(System.currentTimeMillis()) - .build(), - Settings.EMPTY - ); - } - - public void testStandAloneTimeSeriesWithSum() throws IOException { - TimeSeriesAggregationBuilder aggregationBuilder = new TimeSeriesAggregationBuilder("ts").subAggregation(sum("sum").field("val1")); - long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2021-01-01T00:00:00Z"); - timeSeriesTestCase(aggregationBuilder, new MatchAllDocsQuery(), iw -> { - writeTS(iw, startTime + 1, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] { "val1", 1 }); - writeTS(iw, startTime + 2, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] { "val1", 2 }); - writeTS(iw, startTime + 3, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 3 }); - writeTS(iw, startTime + 4, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 4 }); - writeTS(iw, startTime + 5, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] { "val1", 5 }); - writeTS(iw, startTime + 6, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] { "val1", 6 }); - writeTS(iw, startTime + 7, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 7 }); - writeTS(iw, startTime + 8, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 8 }); - }, ts -> { - assertThat(ts.getBuckets(), hasSize(3)); - - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); - assertThat( - ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").getAggregations().get("sum")).value(), - equalTo(6.0) - ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); - assertThat( - ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").getAggregations().get("sum")).value(), - equalTo(8.0) - ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); - assertThat( - ((Sum) ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").getAggregations().get("sum")).value(), - equalTo(22.0) - ); - - }, - new KeywordFieldMapper.KeywordFieldType("dim1"), - new KeywordFieldMapper.KeywordFieldType("dim2"), - new NumberFieldMapper.NumberFieldType("val1", NumberFieldMapper.NumberType.INTEGER) - ); - } - - public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimensions, Object[] metrics) throws IOException { - final List fields = new ArrayList<>(); - fields.add(new SortedNumericDocValuesField(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp)); - fields.add(new LongPoint(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp)); - final TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(null); - for (int i = 0; i < dimensions.length; i += 2) { - if (dimensions[i + 1] instanceof Number n) { - builder.addLong(dimensions[i].toString(), n.longValue()); - } else { - builder.addString(dimensions[i].toString(), dimensions[i + 1].toString()); - } - } - for (int i = 0; i < metrics.length; i += 2) { - if (metrics[i + 1] instanceof Integer || metrics[i + 1] instanceof Long) { - fields.add(new NumericDocValuesField(metrics[i].toString(), ((Number) metrics[i + 1]).longValue())); - } else if (metrics[i + 1] instanceof Float) { - fields.add(new FloatDocValuesField(metrics[i].toString(), (float) metrics[i + 1])); - } else if (metrics[i + 1] instanceof Double) { - fields.add(new DoubleDocValuesField(metrics[i].toString(), (double) metrics[i + 1])); - } - } - fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.withHash().toBytesRef())); - // TODO: Handle metrics - iw.addDocument(fields); - } - - public void testWithDateHistogramExecutedAsFilterByFilterWithTimeSeriesIndexSearcher() throws IOException { - DateHistogramAggregationBuilder aggregationBuilder = new DateHistogramAggregationBuilder("by_timestamp").field("@timestamp") - .fixedInterval(DateHistogramInterval.HOUR) - .subAggregation(new TimeSeriesAggregationBuilder("ts").subAggregation(sum("sum").field("val1"))); - - // Before this threw a CollectionTerminatedException because FilterByFilterAggregation#getLeafCollector() always returns a - // LeafBucketCollector.NO_OP_COLLECTOR instance. And TimeSeriesIndexSearcher can't deal with this when initializing the - // leaf walkers. - testCase(iw -> { - long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); - for (int i = 1; i <= 5000; i++) { - writeTS(iw, startTime++, new Object[] { "dim1", "aaa" }, new Object[] { "val1", 1 }); - } - }, internalAggregation -> { - InternalDateHistogram dateHistogram = (InternalDateHistogram) internalAggregation; - assertThat(dateHistogram.getBuckets(), hasSize(1)); - InternalTimeSeries timeSeries = dateHistogram.getBuckets().get(0).getAggregations().get("ts"); - assertThat(timeSeries.getBuckets(), hasSize(1)); - Sum sum = timeSeries.getBuckets().get(0).getAggregations().get("sum"); - assertThat(sum.value(), equalTo(5000.0)); - }, - new AggTestConfig( - aggregationBuilder, - TimeSeriesIdFieldMapper.FIELD_TYPE, - new DateFieldMapper.DateFieldType("@timestamp"), - new KeywordFieldMapper.KeywordFieldType("dim1"), - new NumberFieldMapper.NumberFieldType("val1", NumberFieldMapper.NumberType.INTEGER) - ).withQuery(new MatchAllDocsQuery()) - ); - } - - public void testMultiBucketAggregationAsSubAggregation() throws IOException { - long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); - CheckedConsumer buildIndex = iw -> { - writeTS(iw, startTime + 1, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] {}); - writeTS(iw, startTime + 2, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] {}); - writeTS(iw, startTime + 3, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] {}); - writeTS(iw, startTime + 4, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] {}); - writeTS(iw, startTime + 5, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] {}); - writeTS(iw, startTime + 6, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] {}); - writeTS(iw, startTime + 7, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] {}); - writeTS(iw, startTime + 8, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] {}); - }; - Consumer verifier = ts -> { - assertThat(ts.getBuckets(), hasSize(3)); - - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L)); - InternalDateHistogram byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg") - .getAggregations() - .get("by_timestamp"); - assertThat( - byTimeStampBucket.getBuckets(), - contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) - ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L)); - byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4") - .getAggregations() - .get("by_timestamp"); - assertThat( - byTimeStampBucket.getBuckets(), - contains(new InternalDateHistogram.Bucket(startTime, 2, false, null, InternalAggregations.EMPTY)) - ); - assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L)); - byTimeStampBucket = ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o") - .getAggregations() - .get("by_timestamp"); - assertThat( - byTimeStampBucket.getBuckets(), - contains(new InternalDateHistogram.Bucket(startTime, 4, false, null, InternalAggregations.EMPTY)) - ); - }; - - DateHistogramAggregationBuilder dateBuilder = new DateHistogramAggregationBuilder("by_timestamp"); - dateBuilder.field("@timestamp"); - dateBuilder.fixedInterval(DateHistogramInterval.seconds(1)); - TimeSeriesAggregationBuilder tsBuilder = new TimeSeriesAggregationBuilder("by_tsid"); - tsBuilder.subAggregation(dateBuilder); - timeSeriesTestCase(tsBuilder, new MatchAllDocsQuery(), buildIndex, verifier); - } - - public void testAggregationSize() throws IOException { - CheckedConsumer buildIndex = multiTsWriter(); - - List> verifiers = new ArrayList>(); - - verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzIS_0VrZGalylFPi9dkK4dYyY9g0yybS6o").docCount, equalTo(4L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyfgM9Vvd-IRpsCvXNT5j_dX4tz0qg").docCount, equalTo(2L))); - verifiers.add(ts -> assertThat(ts.getBucketByKey("KAtLjIWbi0EqQEOYqU2RSzJjzNyftxCMQuGv-XOPz9J6bZM-ZhUGnV4").docCount, equalTo(2L))); - - for (int i = 1; i <= 3; i++) { - int size = i; - Consumer limitedVerifier = ts -> { - assertThat(ts.getBuckets(), hasSize(size)); - - for (int j = 0; j < size; j++) { - verifiers.get(j).accept(ts); - } - }; - - TimeSeriesAggregationBuilder limitedTsBuilder = new TimeSeriesAggregationBuilder("by_tsid"); - limitedTsBuilder.setSize(i); - timeSeriesTestCase(limitedTsBuilder, new MatchAllDocsQuery(), buildIndex, limitedVerifier); - } - } - - private CheckedConsumer multiTsWriter() { - long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); - return iw -> { - writeTS(iw, startTime + 1, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] { "val1", 1 }); - writeTS(iw, startTime + 2, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] { "val1", 2 }); - writeTS(iw, startTime + 3, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 3 }); - writeTS(iw, startTime + 4, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 4 }); - writeTS(iw, startTime + 5, new Object[] { "dim1", "aaa", "dim2", "xxx" }, new Object[] { "val1", 5 }); - writeTS(iw, startTime + 6, new Object[] { "dim1", "aaa", "dim2", "yyy" }, new Object[] { "val1", 6 }); - writeTS(iw, startTime + 7, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 7 }); - writeTS(iw, startTime + 8, new Object[] { "dim1", "bbb", "dim2", "zzz" }, new Object[] { "val1", 8 }); - }; - } - - private void timeSeriesTestCase( - TimeSeriesAggregationBuilder builder, - Query query, - CheckedConsumer buildIndex, - Consumer verify, - MappedFieldType... fieldTypes - ) throws IOException { - MappedFieldType[] newFieldTypes = new MappedFieldType[fieldTypes.length + 2]; - newFieldTypes[0] = TimeSeriesIdFieldMapper.FIELD_TYPE; - newFieldTypes[1] = new DateFieldMapper.DateFieldType("@timestamp"); - System.arraycopy(fieldTypes, 0, newFieldTypes, 2, fieldTypes.length); - - testCase(buildIndex, verify, new AggTestConfig(builder, newFieldTypes).withQuery(query)); - } - -} diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index b645d20aa31e2..33ca0dc4779b6 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -54,8 +54,8 @@ setup: --- "Basic test": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.6.99" + reason: Time series result serialization changed in 8.6.0 - do: search: @@ -76,7 +76,7 @@ setup: - match: { hits.total.value: 1 } - length: { aggregations: 1 } - - match: { aggregations.ts.buckets.0.key: "JEMc3XgEDWzF216sBF5nxwWv9hou4__k9oZfeombF5dLPwkqUA" } + - match: { aggregations.ts.buckets.0.key: { "key": "foo" } } - match: { aggregations.ts.buckets.0.doc_count: 1 } --- @@ -107,8 +107,8 @@ setup: --- "Size test": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.6.99" + reason: Size added in 8.7.0 - do: search: @@ -126,7 +126,7 @@ setup: size: 1 - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: "JEMc3XgEDWzF216sBF5nxwWKf_6WZzMy4zeMjdIBoQC3xHGkjg" } + - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } - do: search: @@ -144,9 +144,9 @@ setup: size: 3 - length: { aggregations.ts.buckets: 3 } - - match: { aggregations.ts.buckets.0.key: "JEMc3XgEDWzF216sBF5nxwWKf_6WZzMy4zeMjdIBoQC3xHGkjg" } - - match: { aggregations.ts.buckets.1.key: "JEMc3XgEDWzF216sBF5nxwWv9hou4__k9oZfeombF5dLPwkqUA" } - - match: { aggregations.ts.buckets.2.key: "JEMc3XgEDWzF216sBF5nxwW5AZB2-M4K_VYiHrKDjUNX3zHVkw" } + - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } + - match: { aggregations.ts.buckets.1.key: { "key": "baz" } } + - match: { aggregations.ts.buckets.2.key: { "key": "foo" } } --- "Score test filter some": @@ -311,8 +311,8 @@ setup: --- "Number for keyword routing field": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.10.99" + reason: "Fix in 8.11" - do: bulk: @@ -341,5 +341,5 @@ setup: - match: { hits.total.value: 2 } - length: { aggregations: 1 } - - match: { aggregations.ts.buckets.0.key: "JEMc3XgEDWzF216sBF5nxwW6KPQ_9D8hCF3DOnGNi7WMdDeQmw" } + - match: { aggregations.ts.buckets.0.key: { "key": "10" } } - match: { aggregations.ts.buckets.0.doc_count: 1 } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 7ef29ab3f1f75..86132b6e29520 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -13,6 +13,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; import org.elasticsearch.cluster.routing.IndexRouting; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.hash.Murmur3Hasher; import org.elasticsearch.common.hash.MurmurHash3; @@ -40,6 +41,8 @@ import java.util.Base64; import java.util.Collections; import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; @@ -338,4 +341,39 @@ private static String base64Encode(final BytesRef bytesRef) { System.arraycopy(bytesRef.bytes, 0, bytes, 0, bytesRef.length); return BASE64_ENCODER.encodeToString(bytes); } + + public static Map decodeTsid2(BytesRef bytesRef) { + try (StreamInput input = new BytesArray(bytesRef).streamInput()) { + return decodeTsid2(input); + } catch (IOException ex) { + throw new IllegalArgumentException("Dimension field cannot be deserialized.", ex); + } + } + + public static Map decodeTsid2(StreamInput in) { + try { + int size = in.readVInt(); + Map result = new LinkedHashMap<>(size); + + for (int i = 0; i < size; i++) { + String name = in.readBytesRef().utf8ToString(); + + int type = in.read(); + switch (type) { + case (byte) 's' -> // parse a string + result.put(name, in.readBytesRef().utf8ToString()); + case (byte) 'l' -> // parse a long + result.put(name, in.readLong()); + case (byte) 'u' -> { // parse an unsigned_long + Object ul = DocValueFormat.UNSIGNED_LONG_SHIFTED.format(in.readLong()); + result.put(name, ul); + } + default -> throw new IllegalArgumentException("Cannot parse [" + name + "]: Unknown type [" + type + "]"); + } + } + return result; + } catch (IOException | IllegalArgumentException e) { + throw new IllegalArgumentException("Error formatting " + NAME + ": " + e.getMessage(), e); + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index c4806dbd3a0a8..c5116dda60441 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -59,6 +59,7 @@ import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xcontent.XContentParserConfiguration; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -330,6 +331,19 @@ public boolean isMultiField(String field) { return mapperService.isMultiField(field); } + public Iterable dimensionFields() { + List dimensionFields = new ArrayList<>(); + for (var mapper : mapperService.mappingLookup().fieldMappers()) { + if (mapper instanceof FieldMapper fieldMapper) { + var fieldType = fieldMapper.fieldType(); + if (fieldType.isDimension()) { + dimensionFields.add(fieldType); + } + } + } + return dimensionFields; + } + public Set sourcePath(String fullName) { return mappingLookup.sourcePaths(fullName); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationExecutionContext.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationExecutionContext.java index 273df99f6479c..14c0f5289c48f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationExecutionContext.java @@ -44,7 +44,7 @@ public LeafReaderContext getLeafReaderContext() { return leafReaderContext; } - public BytesRef getTsid() { + public BytesRef getTsidHash() { return tsidProvider != null ? tsidProvider.get() : null; } @@ -52,7 +52,7 @@ public long getTimestamp() { return timestampProvider.getAsLong(); } - public int getTsidOrd() { + public int getTsidHashOrd() { if (tsidOrdProvider == null) { throw new IllegalArgumentException( "Aggregation on a time-series field is misconfigured, likely due to lack of wrapping " diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java index 4acf66ff979ab..03913717992c9 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/TimeSeriesIndexSearcherTests.java @@ -242,7 +242,7 @@ public void collect(int doc, long owningBucketOrd) throws IOException { assertTrue(timestamp.advanceExact(doc)); BytesRef latestTSID = tsid.lookupOrd(tsid.ordValue()); long latestTimestamp = timestamp.longValue(); - assertEquals(latestTSID, aggCtx.getTsid()); + assertEquals(latestTSID, aggCtx.getTsidHash()); assertEquals(latestTimestamp, aggCtx.getTimestamp()); if (currentTSID != null) { @@ -255,14 +255,14 @@ public void collect(int doc, long owningBucketOrd) throws IOException { currentTimestamp + "->" + latestTimestamp, timestampReverse ? latestTimestamp <= currentTimestamp : latestTimestamp >= currentTimestamp ); - assertEquals(currentTSIDord, aggCtx.getTsidOrd()); + assertEquals(currentTSIDord, aggCtx.getTsidHashOrd()); } else { - assertThat(aggCtx.getTsidOrd(), greaterThan(currentTSIDord)); + assertThat(aggCtx.getTsidHashOrd(), greaterThan(currentTSIDord)); } } currentTimestamp = latestTimestamp; currentTSID = BytesRef.deepCopyOf(latestTSID); - currentTSIDord = aggCtx.getTsidOrd(); + currentTSIDord = aggCtx.getTsidHashOrd(); total++; } }; diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 3615686674bab..326fa46abc4a3 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -374,7 +374,14 @@ public void onCache(ShardId shardId, Accountable accountable) {} () -> true, valuesSourceRegistry, emptyMap() - ); + ) { + @Override + public Iterable dimensionFields() { + return Arrays.stream(fieldTypes) + .filter(MappedFieldType::isDimension) + .toList(); + } + }; AggregationContext context = new ProductionAggregationContext( Optional.ofNullable(analysisModule).map(AnalysisModule::getAnalysisRegistry).orElse(null), diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java index 5eefa7cfc56fa..f153a397f9f58 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregator.java @@ -120,10 +120,10 @@ public void collect(int doc, long bucket) throws IOException { startTimes = bigArrays().grow(startTimes, bucket + 1); endTimes = bigArrays().grow(endTimes, bucket + 1); resetCompensations = bigArrays().grow(resetCompensations, bucket + 1); - if (currentTsid != aggCtx.getTsidOrd()) { + if (currentTsid != aggCtx.getTsidHashOrd()) { // if we're on a new tsid then we need to calculate the last bucket calculateLastBucket(); - currentTsid = aggCtx.getTsidOrd(); + currentTsid = aggCtx.getTsidHashOrd(); } else { // if we're in a new bucket but in the same tsid then we update the // timestamp and last value before we calculate the last bucket diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java index 64da8c8a68080..6137ba23d9fe3 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java @@ -362,12 +362,12 @@ public LeafBucketCollector getLeafCollector(final AggregationExecutionContext ag @Override public void collect(int docId, long owningBucketOrd) throws IOException { task.addNumReceived(1); - final BytesRef tsid = aggCtx.getTsid(); - assert tsid != null : "Document without [" + TimeSeriesIdFieldMapper.NAME + "] field was found."; - final int tsidOrd = aggCtx.getTsidOrd(); + final BytesRef tsidHash = aggCtx.getTsidHash(); + assert tsidHash != null : "Document without [" + TimeSeriesIdFieldMapper.NAME + "] field was found."; + final int tsidHashOrd = aggCtx.getTsidHashOrd(); final long timestamp = timestampField.resolution().roundDownToMillis(aggCtx.getTimestamp()); - boolean tsidChanged = tsidOrd != downsampleBucketBuilder.tsidOrd(); + boolean tsidChanged = tsidHashOrd != downsampleBucketBuilder.tsidOrd(); if (tsidChanged || timestamp < lastHistoTimestamp) { lastHistoTimestamp = Math.max( rounding.round(timestamp), @@ -381,7 +381,7 @@ public void collect(int docId, long owningBucketOrd) throws IOException { logger.trace( "Doc: [{}] - _tsid: [{}], @timestamp: [{}}] -> downsample bucket ts: [{}]", docId, - DocValueFormat.TIME_SERIES_ID.format(tsid), + DocValueFormat.TIME_SERIES_ID.format(tsidHash), timestampFormat.format(timestamp), timestampFormat.format(lastHistoTimestamp) ); @@ -393,13 +393,13 @@ public void collect(int docId, long owningBucketOrd) throws IOException { * - @timestamp must be sorted in descending order within the same _tsid */ BytesRef lastTsid = downsampleBucketBuilder.tsid(); - assert lastTsid == null || lastTsid.compareTo(tsid) <= 0 + assert lastTsid == null || lastTsid.compareTo(tsidHash) <= 0 : "_tsid is not sorted in ascending order: [" + DocValueFormat.TIME_SERIES_ID.format(lastTsid) + "] -> [" - + DocValueFormat.TIME_SERIES_ID.format(tsid) + + DocValueFormat.TIME_SERIES_ID.format(tsidHash) + "]"; - assert tsid.equals(lastTsid) == false || lastTimestamp >= timestamp + assert tsidHash.equals(lastTsid) == false || lastTimestamp >= timestamp : "@timestamp is not sorted in descending order: [" + timestampFormat.format(lastTimestamp) + "] -> [" @@ -416,7 +416,7 @@ public void collect(int docId, long owningBucketOrd) throws IOException { // Create new downsample bucket if (tsidChanged) { - downsampleBucketBuilder.resetTsid(tsid, tsidOrd, lastHistoTimestamp); + downsampleBucketBuilder.resetTsid(tsidHash, tsidHashOrd, lastHistoTimestamp); } else { downsampleBucketBuilder.resetTimestamp(lastHistoTimestamp); } From 44d65218eae5e9a28b533d97e55d22430b2c4ba3 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 30 Nov 2023 13:40:13 +0100 Subject: [PATCH 073/125] iter --- .../bucket/timeseries/InternalTimeSeries.java | 2 +- .../timeseries/TimeSeriesAggregator.java | 19 +++++++++---------- .../index/mapper/TimeSeriesIdFieldMapper.java | 6 +++--- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java index 9b80002416c6b..ea34219adb721 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java @@ -82,7 +82,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public Map getKey() { - return TimeSeriesIdFieldMapper.decodeTsid2(key); + return TimeSeriesIdFieldMapper.decodeTsidAsMap(key); } @Override diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java index da4cf426c5250..fdaaec0eb1be2 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java @@ -106,19 +106,14 @@ protected void doClose() { Releasables.close(bucketOrds); } - @FunctionalInterface - interface TsidConsumer { - void accept(int docId, TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder) throws IOException; - } - @Override protected LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCtx, LeafBucketCollector sub) throws IOException { - SortedMap map = new TreeMap<>(); + SortedMap dimensionConsumers = new TreeMap<>(); for (var entry : dimensionValueSources.entrySet()) { String fieldName = entry.getKey(); if (entry.getValue() instanceof ValuesSource.Numeric numericVS) { SortedNumericDocValues docValues = numericVS.longValues(aggCtx.getLeafReaderContext()); - map.put(entry.getKey(), (docId, tsidBuilder) -> { + dimensionConsumers.put(entry.getKey(), (docId, tsidBuilder) -> { if (docValues.advanceExact(docId)) { for (int i = 0; i < docValues.docValueCount(); i++) { tsidBuilder.addLong(fieldName, docValues.nextValue()); @@ -127,7 +122,7 @@ protected LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCt }); } else { SortedBinaryDocValues docValues = entry.getValue().bytesValues(aggCtx.getLeafReaderContext()); - map.put(entry.getKey(), (docId, tsidBuilder) -> { + dimensionConsumers.put(entry.getKey(), (docId, tsidBuilder) -> { if (docValues.advanceExact(docId)) { for (int i = 0; i < docValues.docValueCount(); i++) { tsidBuilder.addString(fieldName, docValues.nextValue()); @@ -156,12 +151,11 @@ public void collect(int doc, long bucket) throws IOException { } TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); - for (TsidConsumer consumer : map.values()) { + for (TsidConsumer consumer : dimensionConsumers.values()) { consumer.accept(doc, tsidBuilder); } BytesRef tsid = tsidBuilder.withoutHash().toBytesRef(); - // TimeSeriesIdFieldMapper.decodeTsid2(tsid); long bucketOrdinal = bucketOrds.add(bucket, tsid); if (bucketOrdinal < 0) { // already seen bucketOrdinal = -1 - bucketOrdinal; @@ -181,4 +175,9 @@ public void collect(int doc, long bucket) throws IOException { InternalTimeSeries buildResult(InternalTimeSeries.InternalBucket[] topBuckets) { return new InternalTimeSeries(name, List.of(topBuckets), keyed, metadata()); } + + @FunctionalInterface + interface TsidConsumer { + void accept(int docId, TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder) throws IOException; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 86132b6e29520..9c7d71eda857f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -342,15 +342,15 @@ private static String base64Encode(final BytesRef bytesRef) { return BASE64_ENCODER.encodeToString(bytes); } - public static Map decodeTsid2(BytesRef bytesRef) { + public static Map decodeTsidAsMap(BytesRef bytesRef) { try (StreamInput input = new BytesArray(bytesRef).streamInput()) { - return decodeTsid2(input); + return decodeTsidAsMap(input); } catch (IOException ex) { throw new IllegalArgumentException("Dimension field cannot be deserialized.", ex); } } - public static Map decodeTsid2(StreamInput in) { + public static Map decodeTsidAsMap(StreamInput in) { try { int size = in.readVInt(); Map result = new LinkedHashMap<>(size); From 1de5dc9ace92e020555f8e2a5ef8362c60bdf68c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 30 Nov 2023 14:25:17 +0100 Subject: [PATCH 074/125] fix: geoline yaml and code style --- .../aggregations/AggregatorTestCase.java | 4 +- .../test/spatial/120_position_geo_line.yml | 136 +++++++++--------- 2 files changed, 69 insertions(+), 71 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 326fa46abc4a3..c8f0a3fcf5ff2 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -377,9 +377,7 @@ public void onCache(ShardId shardId, Accountable accountable) {} ) { @Override public Iterable dimensionFields() { - return Arrays.stream(fieldTypes) - .filter(MappedFieldType::isDimension) - .toList(); + return Arrays.stream(fieldTypes).filter(MappedFieldType::isDimension).toList(); } }; diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml index 85e1e1a5800ad..580505ba89bc8 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/120_position_geo_line.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.7.99" + reason: position metric introduced in 8.8.0 - do: indices.create: index: locations @@ -87,8 +87,8 @@ setup: - length: { aggregations.by_time_series.buckets: 3 } - match: aggregations.by_time_series.buckets: - "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": - key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" + "{city=Paris}": + key: { city: Paris } doc_count: 2 museum_tour: type: Feature @@ -96,22 +96,22 @@ setup: coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] type: LineString properties: { complete: true } - "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": - key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" - doc_count: 5 + "{city=Antwerp}": + key: { city: Antwerp } + doc_count: 3 museum_tour: type: Feature geometry: - coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] + coordinates: [ [ 4.401384, 51.220292 ], [ 4.405819, 51.221758 ], [ 4.4052, 51.2229 ] ] type: LineString properties: { complete: true } - "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": - key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" - doc_count: 3 + "{city=Amsterdam}": + key: { city: Amsterdam } + doc_count: 5 museum_tour: type: Feature geometry: - coordinates: [ [ 4.401384, 51.220292 ], [ 4.405819, 51.221758 ], [ 4.4052, 51.2229 ] ] + coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] type: LineString properties: { complete: true } @@ -140,8 +140,8 @@ setup: - length: { aggregations.by_time_series.buckets: 3 } - match: aggregations.by_time_series.buckets: - "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": - key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" + "{city=Paris}": + key: { city: Paris } doc_count: 2 museum_tour: type: Feature @@ -149,8 +149,17 @@ setup: coordinates: [ [ 2.327, 48.86 ], [ 2.336389, 48.861111 ] ] type: LineString properties: { complete: true } - "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": - key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" + "{city=Antwerp}": + key: { city: Antwerp } + doc_count: 3 + museum_tour: + type: Feature + geometry: + coordinates: [ [ 4.4052, 51.2229 ], [ 4.405819, 51.221758 ], [ 4.401384, 51.220292 ] ] + type: LineString + properties: { complete: true } + "{city=Amsterdam}": + key: { city: Amsterdam } doc_count: 5 museum_tour: type: Feature @@ -162,15 +171,6 @@ setup: [ 4.889187, 52.373184 ] ] type: LineString properties: { complete: true } - "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": - key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" - doc_count: 3 - museum_tour: - type: Feature - geometry: - coordinates: [ [ 4.4052, 51.2229 ], [ 4.405819, 51.221758 ], [ 4.401384, 51.220292 ] ] - type: LineString - properties: { complete: true } --- "geo_line on position field time time-series with invalid sort specified": @@ -216,8 +216,8 @@ setup: - length: { aggregations.by_time_series.buckets: 3 } - match: aggregations.by_time_series.buckets: - "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": - key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" + "{city=Paris}": + key: { city: Paris } doc_count: 2 museum_tour: type: Feature @@ -225,22 +225,22 @@ setup: coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] type: LineString properties: { complete: true } - "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": - key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" - doc_count: 5 + "{city=Antwerp}": + key: { city: Antwerp } + doc_count: 3 museum_tour: type: Feature geometry: - coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] + coordinates: [ [ 4.401384, 51.220292 ], [ 4.405819, 51.221758 ], [ 4.4052, 51.2229 ] ] type: LineString properties: { complete: true } - "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": - key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" - doc_count: 3 + "{city=Amsterdam}": + key: { city: Amsterdam } + doc_count: 5 museum_tour: type: Feature geometry: - coordinates: [ [ 4.401384, 51.220292 ], [ 4.405819, 51.221758 ], [ 4.4052, 51.2229 ] ] + coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] type: LineString properties: { complete: true } @@ -326,40 +326,40 @@ setup: - length: { aggregations.with_time_series.buckets: 3 } - match: aggregations.with_time_series.buckets: - "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": - key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" - doc_count: 2 + "{city=Amsterdam}": + key: { city: Amsterdam } + doc_count: 5 museums: doc_count_error_upper_bound: 0 sum_other_doc_count: 0 buckets: - - key: Paris - doc_count: 2 + - key: Amsterdam + doc_count: 5 museum_tour: type: Feature geometry: - coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] + coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] type: LineString properties: complete: true - "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": - key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" - doc_count: 5 + "{city=Paris}": + key: { city: Paris } + doc_count: 2 museums: doc_count_error_upper_bound: 0 sum_other_doc_count: 0 buckets: - - key: Amsterdam - doc_count: 5 + - key: Paris + doc_count: 2 museum_tour: type: Feature geometry: - coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ], [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] + coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] type: LineString properties: complete: true - "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": - key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" + "{city=Antwerp}": + key: { city: Antwerp } doc_count: 3 museums: doc_count_error_upper_bound: 0 @@ -447,49 +447,49 @@ setup: - length: { aggregations.with_time_series.buckets: 3 } - match: aggregations.with_time_series.buckets: - "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA": - key: "JImYqeT78g3dNlnfZk5blM4STgeFvROwwyPMhSTP4gNXB4HpdA" - doc_count: 2 + "{city=Amsterdam}": + key: { city: Amsterdam } + doc_count: 5 museums: doc_count_error_upper_bound: 0 sum_other_doc_count: 0 buckets: - key: Museum - doc_count: 2 + doc_count: 3 museum_tour: type: Feature geometry: - coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] + coordinates: [ [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] type: LineString properties: complete: true - "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA": - key: "JImYqeT78g3dNlnfZk5blM5NvpgWBU63c01ElqiK35DCiVINgA" - doc_count: 5 - museums: - doc_count_error_upper_bound: 0 - sum_other_doc_count: 0 - buckets: - - key: Museum - doc_count: 3 + - key: Attraction + doc_count: 2 museum_tour: type: Feature geometry: - coordinates: [ [ 4.901618, 52.369219 ], [ 4.91235, 52.374081 ], [ 4.914722, 52.371667 ] ] + coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ] ] type: LineString properties: complete: true - - key: Attraction + "{city=Paris}": + key: { city: Paris } + doc_count: 2 + museums: + doc_count_error_upper_bound: 0 + sum_other_doc_count: 0 + buckets: + - key: Museum doc_count: 2 museum_tour: type: Feature geometry: - coordinates: [ [ 4.889187, 52.373184 ], [ 4.885057, 52.370159 ] ] + coordinates: [ [ 2.336389, 48.861111 ], [ 2.327, 48.86 ] ] type: LineString properties: complete: true - "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw": - key: "JImYqeT78g3dNlnfZk5blM73Ehzlh3t74AgA5ODr5J84bzmAhw" + "{city=Antwerp}": + key: { city: Antwerp } doc_count: 3 museums: doc_count_error_upper_bound: 0 From 0128c9a45570dfb21afaabcbe809c4d647978c54 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 30 Nov 2023 16:01:09 +0100 Subject: [PATCH 075/125] fix: a yaml test and a rate aggregation test --- .../rate/TimeSeriesRateAggregatorTests.java | 36 +++++++++++-------- .../test/analytics/reset_tracking_rate.yml | 5 +-- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index 4b7688209b81f..e15ae5b448cc1 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -11,14 +11,18 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.aggregations.AggregationsPlugin; import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries; import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesParams; @@ -55,18 +59,15 @@ public void testSimple() throws IOException { Consumer verifier = r -> { assertThat(r.getBuckets(), hasSize(2)); assertThat( - ((Rate) r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2LDvTu5MAoxgghmvEUd5a_i3Y3TA").getAggregations().asList().get(0)) - .getValue(), + ((Rate) r.getBucketByKey("{dim=1}").getAggregations().asList().get(0)).getValue(), closeTo(59.0 / 3000.0 * MILLIS_IN_SECOND, 0.00001) ); assertThat( - ((Rate) r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2hRiY7lpxc5PgA4k16syGbZGgr0g").getAggregations().asList().get(0)) - .getValue(), + ((Rate) r.getBucketByKey("{dim=2}").getAggregations().asList().get(0)).getValue(), closeTo(206.0 / 4000.0 * MILLIS_IN_SECOND, 0.00001) ); }; - AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field")) - .withSplitLeavesIntoSeperateAggregators(false); + AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field"), dimensionField("dim")); testCase(iw -> { iw.addDocuments(docs(1000, "1", 15, 37, 60, /*reset*/ 14)); iw.addDocuments(docs(1000, "2", 74, 150, /*reset*/ 50, 90, /*reset*/ 40)); @@ -84,11 +85,8 @@ public void testNestedWithinDateHistogram() throws IOException { Consumer verifier = r -> { assertThat(r.getBuckets(), hasSize(2)); - assertThat( - r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2LDvTu5MAoxgghmvEUd5a_i3Y3TA"), - instanceOf(InternalTimeSeries.InternalBucket.class) - ); - InternalDateHistogram hb = r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2LDvTu5MAoxgghmvEUd5a_i3Y3TA").getAggregations().get("date"); + assertThat(r.getBucketByKey("{dim=1}"), instanceOf(InternalTimeSeries.InternalBucket.class)); + InternalDateHistogram hb = r.getBucketByKey("{dim=1}").getAggregations().get("date"); { Rate rate = hb.getBuckets().get(1).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((60 - 37 + 14) / 2000.0 * MILLIS_IN_SECOND, 0.00001)); @@ -97,7 +95,7 @@ public void testNestedWithinDateHistogram() throws IOException { Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((37 - 15) / 1000.0 * MILLIS_IN_SECOND, 0.00001)); } - hb = r.getBucketByKey("JFFUy14C9UcX3MnFnsFrpf2hRiY7lpxc5PgA4k16syGbZGgr0g").getAggregations().get("date"); + hb = r.getBucketByKey("{dim=2}").getAggregations().get("date"); { Rate rate = hb.getBuckets().get(0).getAggregations().get("counter_field"); assertThat(rate.getValue(), closeTo((150 - 74) / 1000.0 * MILLIS_IN_SECOND, 0.00001)); @@ -108,7 +106,7 @@ public void testNestedWithinDateHistogram() throws IOException { } }; - AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field")) + AggTestConfig aggTestConfig = new AggTestConfig(tsBuilder, timeStampField(), counterField("counter_field"), dimensionField("dim")) .withSplitLeavesIntoSeperateAggregators(false); testCase(iw -> { iw.addDocuments(docs(2000, "1", 15, 37, 60, /*reset*/ 14)); @@ -159,7 +157,7 @@ private List docs(long startTimestamp, String dim, long... values) thr List documents = new ArrayList<>(); for (int i = 0; i < values.length; i++) { - documents.add(doc(startTimestamp + (i * 1000L), tsid(dim), values[i])); + documents.add(doc(startTimestamp + (i * 1000L), tsid(dim), values[i], dim)); } return documents; } @@ -170,14 +168,22 @@ private static BytesReference tsid(String dim) throws IOException { return idBuilder.withHash(); } - private Document doc(long timestamp, BytesReference tsid, long counterValue) { + private Document doc(long timestamp, BytesReference tsid, long counterValue, String dim) { Document doc = new Document(); doc.add(new SortedNumericDocValuesField("@timestamp", timestamp)); doc.add(new SortedDocValuesField("_tsid", tsid.toBytesRef())); doc.add(new NumericDocValuesField("counter_field", counterValue)); + doc.add(new SortedDocValuesField("dim", new BytesRef(dim))); return doc; } + private MappedFieldType dimensionField(String name) { + return new KeywordFieldMapper.Builder(name, IndexVersion.current()).dimension(true) + .docValues(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType(); + } + private MappedFieldType counterField(String name) { return new NumberFieldMapper.NumberFieldType( name, diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml index e6e9a8a0aa295..62115b3b87081 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/reset_tracking_rate.yml @@ -103,9 +103,6 @@ setup: - '{ "key": "bar", "val": 20, "@timestamp": "2021-01-01T02:40:00Z" }' --- "test resets do not lead to negative rate": - - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 - do: search: index: tsdb-* @@ -129,7 +126,7 @@ setup: - match: { hits.total.value: 9 } - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: "JEMc3XgEDWzF216sBF5nxwW5AZB2-M4K_VYiHrKDjUNX3zHVkw" } + - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } - match: { aggregations.ts.buckets.0.doc_count: 9 } - gte: { aggregations.ts.buckets.0.rate.value: 0.0 } From 0bb9f846d0df2f9b9cd6a1ccc87a7588f83641ec Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 1 Dec 2023 11:23:15 +0100 Subject: [PATCH 076/125] fix: geoline test for time series mode --- .../rest-api-spec/test/aggregations/time_series.yml | 4 +++- .../search/aggregations/GeoLineAggregatorTests.java | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 33ca0dc4779b6..392d7a6a1c30b 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -341,5 +341,7 @@ setup: - match: { hits.total.value: 2 } - length: { aggregations: 1 } - - match: { aggregations.ts.buckets.0.key: { "key": "10" } } + - match: { aggregations.ts.buckets.0.key: { "key": "11" } } - match: { aggregations.ts.buckets.0.doc_count: 1 } + - match: { aggregations.ts.buckets.1.key: { "key": "10" } } + - match: { aggregations.ts.buckets.1.doc_count: 1 } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index b367393be2498..24fe7d26f6849 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -38,11 +38,13 @@ import org.elasticsearch.geometry.simplify.SimplificationErrorCalculator; import org.elasticsearch.geometry.simplify.StreamingGeometrySimplifier; import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.plugins.SearchPlugin; @@ -63,7 +65,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -954,7 +955,12 @@ private void testCase( fieldTypes.add(new GeoPointFieldMapper.GeoPointFieldType("value_field")); } fieldTypes.add(new DateFieldMapper.DateFieldType("time_field")); - fieldTypes.add(new KeywordFieldMapper.KeywordFieldType("group_id", false, true, Collections.emptyMap())); + fieldTypes.add( + new KeywordFieldMapper.Builder("group_id", IndexVersion.current()).dimension(true) + .docValues(true) + .build(MapperBuilderContext.root(true, true)) + .fieldType() + ); fieldTypes.add(new NumberFieldMapper.NumberFieldType("sort_field", NumberFieldMapper.NumberType.LONG)); AggTestConfig aggTestConfig = new AggTestConfig(aggregationBuilder, fieldTypes.toArray(new MappedFieldType[0])); From eef35dca34e5b1e8ab44408a56409a8773361247 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 1 Dec 2023 11:55:35 +0100 Subject: [PATCH 077/125] fix: another yaml test --- .../test/aggregations/time_series.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 392d7a6a1c30b..e0069782bb2b2 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -107,8 +107,8 @@ setup: --- "Size test": - skip: - version: " - 8.6.99" - reason: Size added in 8.7.0 + version: " - 8.11.99" + reason: _tsid hashing introduced in 8.12 - do: search: @@ -126,7 +126,7 @@ setup: size: 1 - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } + - match: { aggregations.ts.buckets.0.key: { "key": "baz" } } - do: search: @@ -144,9 +144,9 @@ setup: size: 3 - length: { aggregations.ts.buckets: 3 } - - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } - - match: { aggregations.ts.buckets.1.key: { "key": "baz" } } - - match: { aggregations.ts.buckets.2.key: { "key": "foo" } } + - match: { aggregations.ts.buckets.0.key: { "key": "baz" } } + - match: { aggregations.ts.buckets.1.key: { "key": "foo" } } + - match: { aggregations.ts.buckets.2.key: { "key": "bar" } } --- "Score test filter some": @@ -311,8 +311,8 @@ setup: --- "Number for keyword routing field": - skip: - version: " - 8.10.99" - reason: "Fix in 8.11" + version: " - 8.11.99" + reason: _tsid hashing introduced in 8.12 - do: bulk: From 8f90e58d059650c5c10feccc199f287997a6631a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 1 Dec 2023 11:57:08 +0100 Subject: [PATCH 078/125] fix: disable IndexingIT test for versions before 8.12 --- .../javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java index d5b5e24e2ccde..ecde9bb946504 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java @@ -230,6 +230,7 @@ private void bulk(String index, String valueSuffix, int count) throws IOExceptio public void testTsdb() throws IOException { assumeTrue("indexing time series indices changed in 8.2.0", getOldClusterVersion().onOrAfter(Version.V_8_2_0)); + assumeTrue("tsid hashing introduces in 8.12", getOldClusterVersion().before(Version.V_8_12_0)); StringBuilder bulk = new StringBuilder(); if (isOldCluster()) { From 0bd2dbe8d37f5a0f0b153b8ebc60db20ef5c16a5 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 4 Dec 2023 11:39:38 +0100 Subject: [PATCH 079/125] fix: backward compatibility of time series doc values formatting --- .../elasticsearch/upgrades/IndexingIT.java | 110 +++++++++++++++--- .../elasticsearch/search/DocValueFormat.java | 18 ++- 2 files changed, 106 insertions(+), 22 deletions(-) diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java index ecde9bb946504..438e519eb5c77 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.test.ListMatcher; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -214,6 +215,31 @@ private void bulk(String index, String valueSuffix, int count) throws IOExceptio } private static final List TSDB_DIMS = List.of("6a841a21", "947e4ced", "a4c385a1", "b47a2f4e", "df3145b3"); + + private static final List EXPECTED_TSDB_TSIDS_NODES_0 = List.of( + "JFFUy14C9UcX3MnFnsFrpf0UyQYeoe87sMXUQ025sHhvhKDa4g", + "JFFUy14C9UcX3MnFnsFrpf3X1Mw5gSg0zb_Y50PDlARq0q0ANA" + ); + + private static final List EXPECTED_TSDB_TSIDS_NODES_1 = List.of( + "JFFUy14C9UcX3MnFnsFrpf0UyQYeoe87sMXUQ025sHhvhKDa4g", + "JFFUy14C9UcX3MnFnsFrpf1qe4IAVEfi3wL8wnx3pd_rA41HpA", + "JFFUy14C9UcX3MnFnsFrpf3X1Mw5gSg0zb_Y50PDlARq0q0ANA" + ); + + private static final List EXPECTED_TSDB_TSIDS_NODES_2 = List.of( + "JFFUy14C9UcX3MnFnsFrpf0UyQYeoe87sMXUQ025sHhvhKDa4g", + "JFFUy14C9UcX3MnFnsFrpf1fK0gnGg01X2P0BuJX0wb-7i2pqA", + "JFFUy14C9UcX3MnFnsFrpf1qe4IAVEfi3wL8wnx3pd_rA41HpA", + "JFFUy14C9UcX3MnFnsFrpf3X1Mw5gSg0zb_Y50PDlARq0q0ANA" + ); + private static final List EXPECTED_TSDB_TSIDS_NODES_3 = List.of( + "JFFUy14C9UcX3MnFnsFrpf0UyQYeoe87sMXUQ025sHhvhKDa4g", + "JFFUy14C9UcX3MnFnsFrpf0ayKYYMlhssuNhG-tPuituN3POiA", + "JFFUy14C9UcX3MnFnsFrpf1fK0gnGg01X2P0BuJX0wb-7i2pqA", + "JFFUy14C9UcX3MnFnsFrpf1qe4IAVEfi3wL8wnx3pd_rA41HpA", + "JFFUy14C9UcX3MnFnsFrpf3X1Mw5gSg0zb_Y50PDlARq0q0ANA" + ); private static final long[] TSDB_TIMES; static { String[] times = new String[] { @@ -229,8 +255,8 @@ private void bulk(String index, String valueSuffix, int count) throws IOExceptio } public void testTsdb() throws IOException { - assumeTrue("indexing time series indices changed in 8.2.0", getOldClusterVersion().onOrAfter(Version.V_8_2_0)); - assumeTrue("tsid hashing introduces in 8.12", getOldClusterVersion().before(Version.V_8_12_0)); + final Version oldClusterVersion = getOldClusterVersion(); + assumeTrue("indexing time series indices changed in 8.2.0", oldClusterVersion.onOrAfter(Version.V_8_2_0)); StringBuilder bulk = new StringBuilder(); if (isOldCluster()) { @@ -238,23 +264,54 @@ public void testTsdb() throws IOException { tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[0], TSDB_TIMES[1], 0.1); tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[0], TSDB_TIMES[1], -0.1); bulk("tsdb", bulk.toString()); - assertTsdbAgg(closeTo(215.95, 0.005), closeTo(-215.95, 0.005)); - return; + assertTsdbAgg(oldClusterVersion, EXPECTED_TSDB_TSIDS_NODES_0, closeTo(215.95, 0.005), closeTo(-215.95, 0.005)); } else if (isFirstMixedCluster()) { tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[1], TSDB_TIMES[2], 0.1); tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[1], TSDB_TIMES[2], -0.1); tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[0], TSDB_TIMES[2], 1.1); bulk("tsdb", bulk.toString()); - assertTsdbAgg(closeTo(217.45, 0.005), closeTo(-217.45, 0.005), closeTo(2391.95, 0.005)); - + if (oldClusterVersion.onOrAfter(Version.V_8_12_0)) { + assertTsdbAgg( + oldClusterVersion, + EXPECTED_TSDB_TSIDS_NODES_1, + closeTo(217.45, 0.005), + closeTo(2391.95, 0.005), + closeTo(-217.45, 0.005) + ); + } else { + assertTsdbAgg( + oldClusterVersion, + EXPECTED_TSDB_TSIDS_NODES_1, + closeTo(217.45, 0.005), + closeTo(-217.45, 0.005), + closeTo(2391.95, 0.005) + ); + } } else if (isMixedCluster()) { tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[2], TSDB_TIMES[3], 0.1); tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[2], TSDB_TIMES[3], -0.1); tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[2], TSDB_TIMES[3], 1.1); tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[0], TSDB_TIMES[3], 10); bulk("tsdb", bulk.toString()); - assertTsdbAgg(closeTo(218.95, 0.005), closeTo(-218.95, 0.005), closeTo(2408.45, 0.005), closeTo(21895, 0.5)); - return; + if (oldClusterVersion.onOrAfter(Version.V_8_12_0)) { + assertTsdbAgg( + oldClusterVersion, + EXPECTED_TSDB_TSIDS_NODES_2, + closeTo(218.95, 0.5), + closeTo(21895.0, 0.005), + closeTo(2408.45, 0.005), + closeTo(-218.95, 0.005) + ); + } else { + assertTsdbAgg( + oldClusterVersion, + EXPECTED_TSDB_TSIDS_NODES_2, + closeTo(218.95, 0.005), + closeTo(-218.95, 0.005), + closeTo(2408.45, 0.005), + closeTo(21895, 0.5) + ); + } } else { tsdbBulk(bulk, TSDB_DIMS.get(0), TSDB_TIMES[3], TSDB_TIMES[4], 0.1); tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[3], TSDB_TIMES[4], -0.1); @@ -262,13 +319,27 @@ public void testTsdb() throws IOException { tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[3], TSDB_TIMES[4], 10); tsdbBulk(bulk, TSDB_DIMS.get(4), TSDB_TIMES[0], TSDB_TIMES[4], -5); bulk("tsdb", bulk.toString()); - assertTsdbAgg( - closeTo(220.45, 0.005), - closeTo(-220.45, 0.005), - closeTo(2424.95, 0.005), - closeTo(22045, 0.5), - closeTo(-11022.5, 0.5) - ); + if (oldClusterVersion.onOrAfter(Version.V_8_12_0)) { + assertTsdbAgg( + oldClusterVersion, + EXPECTED_TSDB_TSIDS_NODES_3, + closeTo(220.45, 0.005), + closeTo(-11022.5, 0.5), + closeTo(22045, 0.5), + closeTo(2424.95, 0.005), + closeTo(-220.45, 0.005) + ); + } else { + assertTsdbAgg( + oldClusterVersion, + EXPECTED_TSDB_TSIDS_NODES_3, + closeTo(220.45, 0.005), + closeTo(-220.45, 0.005), + closeTo(2424.95, 0.005), + closeTo(22045, 0.5), + closeTo(-11022.5, 0.5) + ); + } } } @@ -310,13 +381,15 @@ private void tsdbBulk(StringBuilder bulk, String dim, long timeStart, long timeE } } - private void assertTsdbAgg(Matcher... expected) throws IOException { + private void assertTsdbAgg(final Version oldClusterVersion, final List expectedTsids, final Matcher... expected) + throws IOException { + boolean onOrAfterTsidHashingVersion = oldClusterVersion.onOrAfter(Version.V_8_12_0); Request request = new Request("POST", "/tsdb/_search"); request.addParameter("size", "0"); XContentBuilder body = JsonXContent.contentBuilder().startObject(); body.startObject("aggs").startObject("tsids"); { - body.startObject("terms").field("field", "_tsid").endObject(); + body.startObject("terms").field("field", TimeSeriesIdFieldMapper.NAME).endObject(); body.startObject("aggs").startObject("avg"); { body.startObject("avg").field("field", "value").endObject(); @@ -327,7 +400,8 @@ private void assertTsdbAgg(Matcher... expected) throws IOException { request.setJsonEntity(Strings.toString(body.endObject())); ListMatcher tsidsExpected = matchesList(); for (int d = 0; d < expected.length; d++) { - Object key = Map.of("dim", TSDB_DIMS.get(d)); + // NOTE: from Version 8.12.0 on we use tsid hashing for the _tsid field + Object key = onOrAfterTsidHashingVersion ? expectedTsids.get(d) : Map.of("dim", IndexingIT.TSDB_DIMS.get(d)); tsidsExpected = tsidsExpected.item(matchesMap().extraOk().entry("key", key).entry("avg", Map.of("value", expected[d]))); } assertMap( diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 90a2c2996dd4d..af020ed473b40 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper.TimeSeriesIdBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; @@ -677,6 +678,7 @@ public double parseDouble(String value, boolean roundUp, LongSupplier now) { class TimeSeriesIdDocValueFormat implements DocValueFormat { private static final Base64.Encoder BASE64_ENCODER = Base64.getUrlEncoder().withoutPadding(); + private static final Base64.Decoder BASE64_DECODER = Base64.getUrlDecoder(); private TimeSeriesIdDocValueFormat() {} @@ -705,9 +707,17 @@ public Object format(BytesRef value) { // allocating more bytes than actually required. As a result of these additional bytes // the Base 64 encoding might include spurious bytes (typically 0s) which result in // additional unwanted TSID trailing characters. - byte[] bytes = new byte[value.length]; - System.arraycopy(value.bytes, 0, bytes, 0, value.length); - return BASE64_ENCODER.encodeToString(bytes); + try { + // NOTE: if the tsid is a map of dimension key/value pairs (as it was before introducing + // tsid hashing) we just decode the map and return it. + return TimeSeriesIdFieldMapper.decodeTsidAsMap(value); + } catch (IllegalArgumentException iaex) { + // NOTE: if decoding fails it means the tsid is not a map of dimension key/value pairs + // but a result of tsid hahsing. In this case we return its Base64 encoding. + byte[] bytes = new byte[value.length]; + System.arraycopy(value.bytes, 0, bytes, 0, value.length); + return BASE64_ENCODER.encodeToString(bytes); + } } @Override @@ -716,7 +726,7 @@ public BytesRef parseBytesRef(Object value) { return valueAsBytesRef; } if (value instanceof String valueAsString) { - return new BytesRef(Base64.getUrlDecoder().decode(valueAsString)); + return new BytesRef(BASE64_DECODER.decode(valueAsString)); } return parseBytesRefMap(value); } From 08714b36ae4bf961b77bf4e3998a67e95938a910 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 4 Dec 2023 16:19:57 +0100 Subject: [PATCH 080/125] fix: temporarily execute terms aggregation in map mode --- .../search/aggregations/GeoLineAggregatorTests.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index 7f0e38e3c11bf..be9f557ce5714 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -97,6 +97,7 @@ public void testMixedMissingValues() throws IOException { .size(10); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("groups").field("group_id") + .executionHint("map") .subAggregation(lineAggregationBuilder); long lonLat = (((long) GeoEncodingUtils.encodeLongitude(90.0)) << 32) | GeoEncodingUtils.encodeLatitude(45.0) & 0xffffffffL; @@ -147,6 +148,7 @@ public void testMissingGeoPointValueField() throws IOException { .size(10); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("groups").field("group_id") + .executionHint("map") .subAggregation(lineAggregationBuilder); // input @@ -178,6 +180,7 @@ public void testMissingSortField() throws IOException { .size(10); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("groups").field("group_id") + .executionHint("map") .subAggregation(lineAggregationBuilder); testCase(aggregationBuilder, iw -> { @@ -318,6 +321,7 @@ public void testOnePoint() throws IOException { .sort(sortConfig) .size(size); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("groups").field("group_id") + .executionHint("map") .subAggregation(lineAggregationBuilder); double lon = GeoEncodingUtils.decodeLongitude(randomInt()); double lat = GeoEncodingUtils.decodeLatitude(randomInt()); @@ -350,7 +354,7 @@ public void testGeoLine_TSDB() throws IOException { var testConfig = new TestConfig(100, 100, 3, 7, g, sortOrder, useTimestamp); Function aggBuilderFunc = useTimestamp ? gl -> new TimeSeriesAggregationBuilder("ts").subAggregation(gl) - : gl -> new TermsAggregationBuilder("groups").field("group_id").subAggregation(gl); + : gl -> new TermsAggregationBuilder("groups").field("group_id").executionHint("map").subAggregation(gl); assertGeoLine_TSDB(testConfig, aggBuilderFunc, tsx -> { assertThat("Number of groups matches number of buckets", tsx.ts.getBuckets().size(), equalTo(tsx.groups.length)); assertThat("Number of time-series buckets", tsx.ts.getBuckets().size(), equalTo(g)); @@ -395,7 +399,7 @@ public void testGeoLine_TSDB_simplified() throws IOException { var testConfig = new TestConfig(100, 10, 3, 7, g, sortOrder, useTimestamp); Function aggBuilderFunc = useTimestamp ? gl -> new TimeSeriesAggregationBuilder("ts").subAggregation(gl) - : gl -> new TermsAggregationBuilder("groups").field("group_id").subAggregation(gl); + : gl -> new TermsAggregationBuilder("groups").field("group_id").executionHint("map").subAggregation(gl); assertGeoLine_TSDB(testConfig, aggBuilderFunc, tsx -> { assertThat("Number of groups matches number of buckets", tsx.ts.getBuckets().size(), equalTo(tsx.groups.length)); assertThat("Number of time-series buckets", tsx.ts.getBuckets().size(), equalTo(g)); From 1ab06036fc265715b30b7c041ad807a724320172 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 5 Dec 2023 12:46:03 +0100 Subject: [PATCH 081/125] fix: do not index the dimension field --- .../index/mapper/KeywordFieldMapper.java | 5 +++++ .../search/aggregations/GeoLineAggregatorTests.java | 13 +++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index b62113a586bba..fb4f0f7c91a9b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -242,6 +242,11 @@ public Builder dimension(boolean dimension) { return this; } + public Builder indexed(boolean indexed) { + this.indexed.setValue(indexed); + return this; + } + private FieldValues scriptValues() { if (script.get() == null) { return null; diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index be9f557ce5714..f328c5fa5805a 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -97,7 +97,7 @@ public void testMixedMissingValues() throws IOException { .size(10); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("groups").field("group_id") - .executionHint("map") + .subAggregation(lineAggregationBuilder); long lonLat = (((long) GeoEncodingUtils.encodeLongitude(90.0)) << 32) | GeoEncodingUtils.encodeLatitude(45.0) & 0xffffffffL; @@ -148,7 +148,7 @@ public void testMissingGeoPointValueField() throws IOException { .size(10); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("groups").field("group_id") - .executionHint("map") + .subAggregation(lineAggregationBuilder); // input @@ -180,7 +180,7 @@ public void testMissingSortField() throws IOException { .size(10); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("groups").field("group_id") - .executionHint("map") + .subAggregation(lineAggregationBuilder); testCase(aggregationBuilder, iw -> { @@ -321,7 +321,7 @@ public void testOnePoint() throws IOException { .sort(sortConfig) .size(size); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("groups").field("group_id") - .executionHint("map") + .subAggregation(lineAggregationBuilder); double lon = GeoEncodingUtils.decodeLongitude(randomInt()); double lat = GeoEncodingUtils.decodeLatitude(randomInt()); @@ -354,7 +354,7 @@ public void testGeoLine_TSDB() throws IOException { var testConfig = new TestConfig(100, 100, 3, 7, g, sortOrder, useTimestamp); Function aggBuilderFunc = useTimestamp ? gl -> new TimeSeriesAggregationBuilder("ts").subAggregation(gl) - : gl -> new TermsAggregationBuilder("groups").field("group_id").executionHint("map").subAggregation(gl); + : gl -> new TermsAggregationBuilder("groups").field("group_id").subAggregation(gl); assertGeoLine_TSDB(testConfig, aggBuilderFunc, tsx -> { assertThat("Number of groups matches number of buckets", tsx.ts.getBuckets().size(), equalTo(tsx.groups.length)); assertThat("Number of time-series buckets", tsx.ts.getBuckets().size(), equalTo(g)); @@ -399,7 +399,7 @@ public void testGeoLine_TSDB_simplified() throws IOException { var testConfig = new TestConfig(100, 10, 3, 7, g, sortOrder, useTimestamp); Function aggBuilderFunc = useTimestamp ? gl -> new TimeSeriesAggregationBuilder("ts").subAggregation(gl) - : gl -> new TermsAggregationBuilder("groups").field("group_id").executionHint("map").subAggregation(gl); + : gl -> new TermsAggregationBuilder("groups").field("group_id").subAggregation(gl); assertGeoLine_TSDB(testConfig, aggBuilderFunc, tsx -> { assertThat("Number of groups matches number of buckets", tsx.ts.getBuckets().size(), equalTo(tsx.groups.length)); assertThat("Number of time-series buckets", tsx.ts.getBuckets().size(), equalTo(g)); @@ -959,6 +959,7 @@ private void testCase( fieldTypes.add( new KeywordFieldMapper.Builder("group_id", IndexVersion.current()).dimension(true) .docValues(true) + .indexed(false) .build(MapperBuilderContext.root(true, true)) .fieldType() ); From f62886fe6769532352a578cb293b5f0a2ac24657 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 6 Dec 2023 10:40:32 +0100 Subject: [PATCH 082/125] test: temporarily disable flattened fields failing tests --- .../test/tsdb/70_dimension_types.yml | 356 +++++++++--------- 1 file changed, 178 insertions(+), 178 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 74eb8ea9ad4fa..75b1c6388c927 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -72,93 +72,93 @@ keyword dimension: - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} ---- -flattened dimension: - - skip: - features: close_to - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 - - - - do: - indices.create: - index: test - body: - settings: - index: - mode: time_series - routing_path: [uid] - time_series: - start_time: 2021-04-28T00:00:00Z - end_time: 2021-04-29T00:00:00Z - mappings: - properties: - "@timestamp": - type: date - uid: - type: keyword - time_series_dimension: true - deployment: - type: flattened - time_series_dimensions: [ "build.tag", "version.major", "version.minor", "version.patch" ] - - - do: - bulk: - refresh: true - index: test - body: - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.2, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.0, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.4, "deployment": { "build": { "tag": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.3, "deployment": { "build": { "tag": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.6, "deployment": { "build": { "tag": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "tag": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.8, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' - - - is_false: errors - - do: - search: - index: test - body: - size: 0 - aggs: - tsids: - terms: - field: _tsid - order: - _key: asc - aggs: - voltage: - avg: - field: voltage - - - match: { hits.total.value: 8} - - length: { aggregations.tsids.buckets: 4} - - - match: { aggregations.tsids.buckets.0.key: "NCLYECP-GoaIfjk0RBfdlg0oaZ29eRDHR3kQx0fjCuTKddqA4R9ytcYdqdbcoJ8VBuvqFtQ" } - - match: { aggregations.tsids.buckets.0.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} - - - match: { aggregations.tsids.buckets.1.key: "NCLYECP-GoaIfjk0RBfdlg2rnf7qeRDHR3kQx0eLDvTuOYoRw4EeUIKK1KGFPmxqCWQyQJE" } - - match: { aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - - match: { aggregations.tsids.buckets.2.key: "NCLYECP-GoaIfjk0RBfdlg3P1l1UeRDHR3kQx0fjCuTKddqA4ai7CiSuCxA-PhBdLYOXkVY" } - - match: { aggregations.tsids.buckets.2.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - - match: { aggregations.tsids.buckets.3.key: "NCLYECP-GoaIfjk0RBfdlg3gKelzeRDHR3kQx0eLDvTuOYoRw6Wapg_P07YIhdV3NFJDjjE" } - - match: { aggregations.tsids.buckets.3.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.65, error: 0.01 }} +#--- +#flattened dimension: +# - skip: +# features: close_to +# version: " - 8.11.99" +# reason: _tsid hasing introduced in 8.12 +# +# +# - do: +# indices.create: +# index: test +# body: +# settings: +# index: +# mode: time_series +# routing_path: [uid] +# time_series: +# start_time: 2021-04-28T00:00:00Z +# end_time: 2021-04-29T00:00:00Z +# mappings: +# properties: +# "@timestamp": +# type: date +# uid: +# type: keyword +# time_series_dimension: true +# deployment: +# type: flattened +# time_series_dimensions: [ "build.tag", "version.major", "version.minor", "version.patch" ] +# +# - do: +# bulk: +# refresh: true +# index: test +# body: +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.2, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.0, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.4, "deployment": { "build": { "tag": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.3, "deployment": { "build": { "tag": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.6, "deployment": { "build": { "tag": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "tag": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.8, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' +# +# - is_false: errors +# - do: +# search: +# index: test +# body: +# size: 0 +# aggs: +# tsids: +# terms: +# field: _tsid +# order: +# _key: asc +# aggs: +# voltage: +# avg: +# field: voltage +# +# - match: { hits.total.value: 8} +# - length: { aggregations.tsids.buckets: 4} +# +# - match: { aggregations.tsids.buckets.0.key: "NCLYECP-GoaIfjk0RBfdlg0oaZ29eRDHR3kQx0fjCuTKddqA4R9ytcYdqdbcoJ8VBuvqFtQ" } +# - match: { aggregations.tsids.buckets.0.doc_count: 2 } +# - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} +# +# - match: { aggregations.tsids.buckets.1.key: "NCLYECP-GoaIfjk0RBfdlg2rnf7qeRDHR3kQx0eLDvTuOYoRw4EeUIKK1KGFPmxqCWQyQJE" } +# - match: { aggregations.tsids.buckets.1.doc_count: 2 } +# - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} +# +# - match: { aggregations.tsids.buckets.2.key: "NCLYECP-GoaIfjk0RBfdlg3P1l1UeRDHR3kQx0fjCuTKddqA4ai7CiSuCxA-PhBdLYOXkVY" } +# - match: { aggregations.tsids.buckets.2.doc_count: 2 } +# - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} +# +# - match: { aggregations.tsids.buckets.3.key: "NCLYECP-GoaIfjk0RBfdlg3gKelzeRDHR3kQx0eLDvTuOYoRw6Wapg_P07YIhdV3NFJDjjE" } +# - match: { aggregations.tsids.buckets.3.doc_count: 2 } +# - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.65, error: 0.01 }} --- flattened empty dimension: @@ -336,97 +336,97 @@ flattened field missing routing path field: - match: { aggregations.tsids.buckets.5.doc_count: 1 } - close_to: { aggregations.tsids.buckets.5.voltage.value: { value: 6.60, error: 0.01 }} ---- -flattened field misspelled routing path field: - - skip: - features: close_to - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 - - - do: - indices.create: - index: test - body: - settings: - index: - mode: time_series - # NOTE: 'reigion' here is misspelled on purpose - routing_path: [deployment.reigion, deployment.build.tag] - time_series: - start_time: 2021-04-28T00:00:00Z - end_time: 2021-04-29T00:00:00Z - mappings: - properties: - "@timestamp": - type: date - uid: - type: keyword - deployment: - type: flattened - # NOTE: 'reigion' here is misspelled on purpose - time_series_dimensions: [ reigion, build.tag ] - - - do: - bulk: - refresh: true - index: test - body: - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.2, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.0, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.4, "deployment": { "build": { "tag": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.3, "deployment": { "build": { "sha": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.6, "deployment": { "build": { "tag": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "sha": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' - - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.8, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' - - - is_true: errors - - - match: { items.3.index.error.reason: "Error extracting routing: source didn't contain any routing fields" } - - match: { items.5.index.error.reason: "Error extracting routing: source didn't contain any routing fields" } - - - do: - search: - index: test - body: - size: 0 - aggs: - tsids: - terms: - field: _tsid - order: - _key: asc - aggs: - voltage: - avg: - field: voltage - - - match: { hits.total.value: 6 } - - length: { aggregations.tsids.buckets: 4 } - - - match: { aggregations.tsids.buckets.0.key: "JBs0-JZ2yoAg-Lrw35Mu3ysoaZ29egRdyNeHXPSPghDVzguaRg" } - - match: { aggregations.tsids.buckets.0.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} - - - match: { aggregations.tsids.buckets.1.key: "JBs0-JZ2yoAg-Lrw35Mu3yurnf7qs9-VXFZ6jjZCbl_iiXSs7Q" } - - match: { aggregations.tsids.buckets.1.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} - - - match: { aggregations.tsids.buckets.2.key: "JBs0-JZ2yoAg-Lrw35Mu3yvP1l1UlmlXEQVNXrHpUvpn7by0jA" } - - match: { aggregations.tsids.buckets.2.doc_count: 2 } - - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} - - - match: { aggregations.tsids.buckets.3.key: "JBs0-JZ2yoAg-Lrw35Mu3yvgKelz9WSJqzeYh7aza_7yxDXMZA" } - - match: { aggregations.tsids.buckets.3.doc_count: 1 } - - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} +#--- +#flattened field misspelled routing path field: +# - skip: +# features: close_to +# version: " - 8.11.99" +# reason: _tsid hasing introduced in 8.12 +# +# - do: +# indices.create: +# index: test +# body: +# settings: +# index: +# mode: time_series +# # NOTE: 'reigion' here is misspelled on purpose +# routing_path: [deployment.reigion, deployment.build.tag] +# time_series: +# start_time: 2021-04-28T00:00:00Z +# end_time: 2021-04-29T00:00:00Z +# mappings: +# properties: +# "@timestamp": +# type: date +# uid: +# type: keyword +# deployment: +# type: flattened +# # NOTE: 'reigion' here is misspelled on purpose +# time_series_dimensions: [ reigion, build.tag ] +# +# - do: +# bulk: +# refresh: true +# index: test +# body: +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.2, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.0, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.4, "deployment": { "build": { "tag": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.3, "deployment": { "build": { "sha": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.6, "deployment": { "build": { "tag": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "sha": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' +# - '{"index": {}}' +# - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.8, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' +# +# - is_true: errors +# +# - match: { items.3.index.error.reason: "Error extracting routing: source didn't contain any routing fields" } +# - match: { items.5.index.error.reason: "Error extracting routing: source didn't contain any routing fields" } +# +# - do: +# search: +# index: test +# body: +# size: 0 +# aggs: +# tsids: +# terms: +# field: _tsid +# order: +# _key: asc +# aggs: +# voltage: +# avg: +# field: voltage +# +# - match: { hits.total.value: 6 } +# - length: { aggregations.tsids.buckets: 4 } +# +# - match: { aggregations.tsids.buckets.0.key: "JBs0-JZ2yoAg-Lrw35Mu3ysoaZ29egRdyNeHXPSPghDVzguaRg" } +# - match: { aggregations.tsids.buckets.0.doc_count: 1 } +# - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} +# +# - match: { aggregations.tsids.buckets.1.key: "JBs0-JZ2yoAg-Lrw35Mu3yurnf7qs9-VXFZ6jjZCbl_iiXSs7Q" } +# - match: { aggregations.tsids.buckets.1.doc_count: 2 } +# - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} +# +# - match: { aggregations.tsids.buckets.2.key: "JBs0-JZ2yoAg-Lrw35Mu3yvP1l1UlmlXEQVNXrHpUvpn7by0jA" } +# - match: { aggregations.tsids.buckets.2.doc_count: 2 } +# - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} +# +# - match: { aggregations.tsids.buckets.3.key: "JBs0-JZ2yoAg-Lrw35Mu3yvgKelz9WSJqzeYh7aza_7yxDXMZA" } +# - match: { aggregations.tsids.buckets.3.doc_count: 1 } +# - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} --- long dimension: From 84ef189a32bab4d5bdf6d4cf129db121b40b1c29 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 7 Dec 2023 14:57:57 +0100 Subject: [PATCH 083/125] fix: address some review comments --- .../timeseries/TimeSeriesAggregator.java | 2 +- .../timeseries/InternalTimeSeriesTests.java | 2 +- .../timeseries/TimeSeriesAggregatorTests.java | 2 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 22 +++++++++---------- .../elasticsearch/search/DocValueFormat.java | 2 +- .../index/mapper/IdLoaderTests.java | 4 ++-- .../mapper/IdLoaderTsidHashingTests.java | 4 ++-- .../search/DocValueFormatTests.java | 2 +- .../rate/TimeSeriesRateAggregatorTests.java | 2 +- .../aggregations/GeoLineAggregatorTests.java | 2 +- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java index fdaaec0eb1be2..f0774baf4c8c6 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java @@ -155,7 +155,7 @@ public void collect(int doc, long bucket) throws IOException { consumer.accept(doc, tsidBuilder); } - BytesRef tsid = tsidBuilder.withoutHash().toBytesRef(); + BytesRef tsid = tsidBuilder.buildLegacyTsid().toBytesRef(); long bucketOrdinal = bucketOrds.add(bucket, tsid); if (bucketOrdinal < 0) { // already seen bucketOrdinal = -1 - bucketOrdinal; diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java index e7255ecbff395..3529f65781216 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java @@ -53,7 +53,7 @@ private List randomBuckets(boolean keyed, InternalAggregations a builder.addString(entry.getKey(), (String) entry.getValue()); } try { - var key = builder.withoutHash().toBytesRef(); + var key = builder.buildLegacyTsid().toBytesRef(); bucketList.add(new InternalBucket(key, docCount, aggregations, keyed)); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index a12e5e53519d3..54d1c14931d92 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -117,7 +117,7 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens fields.add(new DoubleDocValuesField(metrics[i].toString(), (double) metrics[i + 1])); } } - fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.withoutHash().toBytesRef())); + fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.buildLegacyTsid().toBytesRef())); iw.addDocument(fields); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 9c7d71eda857f..3a525e26e501c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -135,8 +135,8 @@ public void postParse(DocumentParserContext context) throws IOException { final TimeSeriesIdBuilder timeSeriesIdBuilder = (TimeSeriesIdBuilder) context.getDimensions(); final BytesRef timeSeriesId = getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ID_HASHING) - ? timeSeriesIdBuilder.withoutHash().toBytesRef() - : timeSeriesIdBuilder.withHash().toBytesRef(); + ? timeSeriesIdBuilder.buildLegacyTsid().toBytesRef() + : timeSeriesIdBuilder.buildTsidHash().toBytesRef(); context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); TsidExtractingIdFieldMapper.createField(context, timeSeriesIdBuilder.routingBuilder, timeSeriesId); } @@ -172,7 +172,7 @@ public static class TimeSeriesIdBuilder implements DocumentDimensions { public static final int MAX_DIMENSIONS = 512; - private record DimensionDataHolder(BytesRef name, BytesReference value) {} + private record Dimension(BytesRef name, BytesReference value) {} private final Murmur3Hasher tsidHasher = new Murmur3Hasher(0); @@ -181,7 +181,7 @@ private record DimensionDataHolder(BytesRef name, BytesReference value) {} * for generating the _tsid field. The map will be used by {@link TimeSeriesIdFieldMapper} * to build the _tsid field for the document. */ - private final SortedSet dimensions = new TreeSet<>(Comparator.comparing(o -> o.name)); + private final SortedSet dimensions = new TreeSet<>(Comparator.comparing(o -> o.name)); /** * Builds the routing. Used for building {@code _id}. If null then skipped. */ @@ -192,14 +192,14 @@ public TimeSeriesIdBuilder(@Nullable IndexRouting.ExtractFromSource.Builder rout this.routingBuilder = routingBuilder; } - public BytesReference withoutHash() throws IOException { + public BytesReference buildLegacyTsid() throws IOException { if (dimensions.isEmpty()) { throw new IllegalArgumentException("Dimension fields are missing."); } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(dimensions.size()); - for (DimensionDataHolder entry : dimensions) { + for (Dimension entry : dimensions) { out.writeBytesRef(entry.name); entry.value.writeTo(out); } @@ -218,21 +218,21 @@ public BytesReference withoutHash() throws IOException { * The idea is to be able to place 'similar' time series close to each other. Two time series * are considered 'similar' if they share the same dimensions (names and values). */ - public BytesReference withHash() throws IOException { + public BytesReference buildTsidHash() throws IOException { // NOTE: hash all dimension field names int numberOfDimensions = Math.min(MAX_DIMENSIONS, dimensions.size()); int tsidHashIndex = 0; byte[] tsidHash = new byte[16 + 16 + 4 * numberOfDimensions]; tsidHasher.reset(); - for (final DimensionDataHolder dimension : dimensions) { + for (final Dimension dimension : dimensions) { tsidHasher.update(dimension.name.bytes); } tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); // NOTE: concatenate all dimension value hashes up to a certain number of dimensions int tsidHashStartIndex = tsidHashIndex; - for (final DimensionDataHolder dimension : dimensions) { + for (final Dimension dimension : dimensions) { if ((tsidHashIndex - tsidHashStartIndex) >= 4 * numberOfDimensions) break; final BytesRef dimensionValueBytesRef = dimension.value.toBytesRef(); ByteUtils.writeIntLE( @@ -250,7 +250,7 @@ public BytesReference withHash() throws IOException { // NOTE: hash all dimension field allValues tsidHasher.reset(); - for (final DimensionDataHolder dimension : dimensions) { + for (final Dimension dimension : dimensions) { tsidHasher.update(dimension.value.toBytesRef().bytes); } tsidHashIndex = writeHash128(tsidHasher.digestHash(), tsidHash, tsidHashIndex); @@ -324,7 +324,7 @@ public void addUnsignedLong(String fieldName, long value) { } private void add(String fieldName, BytesReference encoded) throws IOException { - final DimensionDataHolder dimension = new DimensionDataHolder(new BytesRef(fieldName), encoded); + final Dimension dimension = new Dimension(new BytesRef(fieldName), encoded); if (dimensions.contains(dimension)) { throw new IllegalArgumentException("Dimension field [" + fieldName + "] cannot be a multi-valued field."); } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 5f81474d91f9e..0d7eae102b700 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -759,7 +759,7 @@ private BytesRef parseBytesRefMap(Object value) { try { // NOTE: we can decode the tsid only if it is not hashed (represented as a map) - return builder.withoutHash().toBytesRef(); + return builder.buildLegacyTsid().toBytesRef(); } catch (IOException e) { throw new IllegalArgumentException(e); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index 6dd1bc799eb8b..5945e5c81856f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -235,7 +235,7 @@ private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter fields.add(new SortedSetDocValuesField(dimension.field, new BytesRef(dimension.value.toString()))); } } - BytesRef tsid = builder.withHash().toBytesRef(); + BytesRef tsid = builder.buildTsidHash().toBytesRef(); fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, tsid)); iw.addDocument(fields); } @@ -253,7 +253,7 @@ private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc return TsidExtractingIdFieldMapper.createId( false, routingBuilder, - timeSeriesIdBuilder.withHash().toBytesRef(), + timeSeriesIdBuilder.buildTsidHash().toBytesRef(), doc.timestamp, new byte[16] ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java index 053432df6cc23..686fb592a7faf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java @@ -234,7 +234,7 @@ private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter fields.add(new SortedSetDocValuesField(dimension.field, new BytesRef(dimension.value.toString()))); } } - BytesRef tsid = builder.withHash().toBytesRef(); + BytesRef tsid = builder.buildTsidHash().toBytesRef(); fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, tsid)); iw.addDocument(fields); } @@ -252,7 +252,7 @@ private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc return TsidExtractingIdFieldMapper.createId( false, routingBuilder, - timeSeriesIdBuilder.withHash().toBytesRef(), + timeSeriesIdBuilder.buildTsidHash().toBytesRef(), doc.timestamp, new byte[16] ); diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index b487f86109edd..8c70b35d27c95 100644 --- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -378,7 +378,7 @@ public void testParseTsid() throws IOException { timeSeriesIdBuilder.addString("string", randomAlphaOfLength(10)); timeSeriesIdBuilder.addLong("long", randomLong()); timeSeriesIdBuilder.addUnsignedLong("ulong", randomLong()); - BytesRef expected = timeSeriesIdBuilder.withHash().toBytesRef(); + BytesRef expected = timeSeriesIdBuilder.buildTsidHash().toBytesRef(); byte[] expectedBytes = new byte[expected.length]; System.arraycopy(expected.bytes, 0, expectedBytes, 0, expected.length); BytesRef actual = DocValueFormat.TIME_SERIES_ID.parseBytesRef(expected); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index e15ae5b448cc1..885e02a8b5e6a 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -165,7 +165,7 @@ private List docs(long startTimestamp, String dim, long... values) thr private static BytesReference tsid(String dim) throws IOException { TimeSeriesIdFieldMapper.TimeSeriesIdBuilder idBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); idBuilder.addString("dim", dim); - return idBuilder.withHash(); + return idBuilder.buildTsidHash(); } private Document doc(long timestamp, BytesReference tsid, long counterValue, String dim) { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index f328c5fa5805a..194a53944d037 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -809,7 +809,7 @@ private void assertGeoLine_TSDB( ArrayList fields = new ArrayList<>( Arrays.asList( new SortedDocValuesField("group_id", new BytesRef(testData.groups[g])), - new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.withHash().toBytesRef()) + new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, builder.buildTsidHash().toBytesRef()) ) ); GeoPoint point = points.get(i); From c53f66bbb7a0a70aca7d70fbaaf2a5046476a13c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 7 Dec 2023 14:58:57 +0100 Subject: [PATCH 084/125] fix: rename method --- .../index/mapper/TimeSeriesIdFieldMapper.java | 4 ++-- .../mapper/TsidExtractingIdFieldMapper.java | 2 +- .../mapper/TimeSeriesIdFieldMapperTests.java | 22 +++++++++---------- ...meSeriesIdFieldMapperTsidHashingTests.java | 22 +++++++++---------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 3a525e26e501c..88f6bbbb563e2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -158,7 +158,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { /** * Decode the {@code _tsid} into a human readable map. */ - public static Object decodeTsid(StreamInput in) { + public static Object encodeTsid(StreamInput in) { try { return base64Encode(in.readBytesRef()); } catch (IOException e) { @@ -332,7 +332,7 @@ private void add(String fieldName, BytesReference encoded) throws IOException { } } - public static Object decodeTsid(final BytesRef bytesRef) { + public static Object encodeTsid(final BytesRef bytesRef) { return base64Encode(bytesRef); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 38a626943fc58..8d248beb97ef1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -204,7 +204,7 @@ public String documentDescription(ParsedDocument parsedDocument) { } private static String tsidDescription(IndexableField tsidField) { - String tsid = TimeSeriesIdFieldMapper.decodeTsid(tsidField.binaryValue()).toString(); + String tsid = TimeSeriesIdFieldMapper.encodeTsid(tsidField.binaryValue()).toString(); if (tsid.length() <= DESCRIPTION_TSID_LIMIT) { return tsid; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index a6e0a7b28936e..d4dc03d22441b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -93,7 +93,7 @@ public void testEnabledInTimeSeriesMode() throws Exception { ); assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); assertThat(doc.rootDoc().getField("b").numericValue(), equalTo(100L)); - assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWE"); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWE"); } public void testDisabledInStandardMode() throws Exception { @@ -138,7 +138,7 @@ public void testStrings() throws IOException { docMapper, b -> b.field("a", "foo").field("b", "bar").field("c", "baz").startObject("o").field("e", "bort").endObject() ); - assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), "AWE"); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWE"); } @SuppressWarnings("unchecked") @@ -151,7 +151,7 @@ public void testUnicodeKeys() throws IOException { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field(fire, "hot").field(coffee, "good")); - Object tsid = TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); + Object tsid = TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); assertEquals(tsid, "A-I"); } @@ -162,7 +162,7 @@ public void testKeywordTooLong() throws IOException { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); - assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); } @SuppressWarnings("unchecked") @@ -173,7 +173,7 @@ public void testKeywordTooLongUtf8() throws IOException { String theWordLong = "長い"; ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); - assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); } public void testKeywordNull() throws IOException { @@ -212,7 +212,7 @@ public void testLong() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", 1234).endObject(); }); - assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testLongInvalidString() throws IOException { @@ -264,7 +264,7 @@ public void testInteger() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", Integer.MIN_VALUE).endObject(); }); - assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testIntegerInvalidString() throws IOException { @@ -320,7 +320,7 @@ public void testShort() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", Short.MIN_VALUE).endObject(); }); - assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testShortInvalidString() throws IOException { @@ -376,7 +376,7 @@ public void testByte() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", (int) Byte.MIN_VALUE).endObject(); }); - assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testByteInvalidString() throws IOException { @@ -432,7 +432,7 @@ public void testIp() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", "255.255.255.1").endObject(); }); - assertEquals(TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWFz"); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWFz"); } public void testIpInvalidString() throws IOException { @@ -467,7 +467,7 @@ public void testVeryLarge() throws IOException { } }); - Object tsid = TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); + Object tsid = TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); assertEquals( tsid, "AWJzA2ZvbwJkMHPwBm1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd" diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java index 2d920d35a35d9..dd6a5504892e2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java @@ -85,7 +85,7 @@ public void testEnabledInTimeSeriesMode() throws Exception { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100).field("c", 500)); assertEquals( "30CO74tuRyatuMXEvNJvbu4OOPE_BPY6XRQDNqrFfRGnHvkCRkdF1g", - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) + TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) ); } @@ -133,7 +133,7 @@ public void testStrings() throws IOException { ); assertEquals( "S3oqtElkKwgW-mWA1nr9gK_2Gi60D5a0AuSISsBYPmjw-qWQ7YI-3A", - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()) + TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()) ); } @@ -149,7 +149,7 @@ public void testUnicodeKeys() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field(fire, "hot").field(coffee, "good")); assertEquals( "A-nLhW-M69syDAiaaHLtCp6MBeMMqVDqgBVXpLViGUOdgCAZniSp6A", - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) + TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) ); } @@ -162,7 +162,7 @@ public void testKeywordTooLong() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); assertEquals( "FlVJQe85E3Mv4Ekz_gxSa71kNSgI7Z7KeQP6VlsQZzmG4ohj", - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) + TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) ); } @@ -176,7 +176,7 @@ public void testKeywordTooLongUtf8() throws IOException { ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); assertEquals( "FlVJQe85E3Mv4Ekz_gxSa0zjd_qfoPhsUMih0si8XDmP_VV7", - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) + TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) ); } @@ -218,7 +218,7 @@ public void testLong() throws IOException { }); assertEquals( "iI8WAECVhaC5BYgDlkz8OK2bjW9KuknL3XBrYDfJ_rolBiDD3_cY-Gu6cUA", - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()) + TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()) ); } @@ -273,7 +273,7 @@ public void testInteger() throws IOException { }); assertEquals( "iI8WAECVhaC5BYgDlkz8OK2bjW9KuknL1fIRUB0-Ns-fIV82SmmaBGvMBFc", - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()) + TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()) ); } @@ -332,7 +332,7 @@ public void testShort() throws IOException { }); assertEquals( "iI8WAECVhaC5BYgDlkz8OK2bjW9KuknLJgBZj8kLWyz4_fSeI5hcDbH2Quo", - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()) + TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()) ); } @@ -391,7 +391,7 @@ public void testByte() throws IOException { }); assertEquals( "iI8WAECVhaC5BYgDlkz8OK2bjW9KuknLKCuqhxFhxsmpAlmUjJPOtj1L020", - TimeSeriesIdFieldMapper.decodeTsid(new BytesArray(tsid).streamInput()) + TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()) ); } @@ -450,7 +450,7 @@ public void testIp() throws IOException { }); assertEquals( "iI8WAECVhaC5BYgDlkz8OAus6VZKuknLs_NO7CBqqZuzDo7PTQiv_wLD1eY", - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) + TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) ); } @@ -492,7 +492,7 @@ public void testVeryLarge() throws IOException { + "Bk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZN" + "YqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKl" + "Bk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTxUG_ass830ZjEXx7y5eiMg", - TimeSeriesIdFieldMapper.decodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) + TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) ); } From 19e50c7b17465fdd9007517188b90cfb479a44af Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 7 Dec 2023 15:00:47 +0100 Subject: [PATCH 085/125] fix: update format comment --- .../org/elasticsearch/search/DocValueFormat.java | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 0d7eae102b700..37cb8e638831b 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -695,22 +695,13 @@ public String toString() { */ @Override public Object format(BytesRef value) { - // NOTE: we need to do this copy to avoid encoding spurious bytes at the end of the - // BytesRef buffer. When sorting results the TermOrdValComparator will copy - // values using a BytesRefBuilder whose underlying bytes buffer is sized incorrectly, - // allocating more bytes than actually required. As a result of these additional bytes - // the Base 64 encoding might include spurious bytes (typically 0s) which result in - // additional unwanted TSID trailing characters. try { // NOTE: if the tsid is a map of dimension key/value pairs (as it was before introducing // tsid hashing) we just decode the map and return it. return TimeSeriesIdFieldMapper.decodeTsidAsMap(value); } catch (IllegalArgumentException iaex) { - // NOTE: if decoding fails it means the tsid is not a map of dimension key/value pairs - // but a result of tsid hahsing. In this case we return its Base64 encoding. - byte[] bytes = new byte[value.length]; - System.arraycopy(value.bytes, 0, bytes, 0, value.length); - return BASE64_ENCODER.encodeToString(bytes); + // NOTE: otherwise the _tsid field is just a hash and we can't decode it + return TimeSeriesIdFieldMapper.encodeTsid(value); } } From 83665805b107b6d81c608f7bc28ac88bebcd41cd Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 7 Dec 2023 15:12:30 +0100 Subject: [PATCH 086/125] fix: _tsid encoding & decoding --- .../test/tsdb/70_dimension_types.yml | 356 +++++++++--------- .../elasticsearch/search/DocValueFormat.java | 2 +- 2 files changed, 179 insertions(+), 179 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 75b1c6388c927..74eb8ea9ad4fa 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -72,93 +72,93 @@ keyword dimension: - match: {aggregations.tsids.buckets.1.doc_count: 4} - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.3, error: 0.01 }} -#--- -#flattened dimension: -# - skip: -# features: close_to -# version: " - 8.11.99" -# reason: _tsid hasing introduced in 8.12 -# -# -# - do: -# indices.create: -# index: test -# body: -# settings: -# index: -# mode: time_series -# routing_path: [uid] -# time_series: -# start_time: 2021-04-28T00:00:00Z -# end_time: 2021-04-29T00:00:00Z -# mappings: -# properties: -# "@timestamp": -# type: date -# uid: -# type: keyword -# time_series_dimension: true -# deployment: -# type: flattened -# time_series_dimensions: [ "build.tag", "version.major", "version.minor", "version.patch" ] -# -# - do: -# bulk: -# refresh: true -# index: test -# body: -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.2, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.0, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.4, "deployment": { "build": { "tag": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.3, "deployment": { "build": { "tag": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.6, "deployment": { "build": { "tag": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "tag": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.8, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' -# -# - is_false: errors -# - do: -# search: -# index: test -# body: -# size: 0 -# aggs: -# tsids: -# terms: -# field: _tsid -# order: -# _key: asc -# aggs: -# voltage: -# avg: -# field: voltage -# -# - match: { hits.total.value: 8} -# - length: { aggregations.tsids.buckets: 4} -# -# - match: { aggregations.tsids.buckets.0.key: "NCLYECP-GoaIfjk0RBfdlg0oaZ29eRDHR3kQx0fjCuTKddqA4R9ytcYdqdbcoJ8VBuvqFtQ" } -# - match: { aggregations.tsids.buckets.0.doc_count: 2 } -# - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} -# -# - match: { aggregations.tsids.buckets.1.key: "NCLYECP-GoaIfjk0RBfdlg2rnf7qeRDHR3kQx0eLDvTuOYoRw4EeUIKK1KGFPmxqCWQyQJE" } -# - match: { aggregations.tsids.buckets.1.doc_count: 2 } -# - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} -# -# - match: { aggregations.tsids.buckets.2.key: "NCLYECP-GoaIfjk0RBfdlg3P1l1UeRDHR3kQx0fjCuTKddqA4ai7CiSuCxA-PhBdLYOXkVY" } -# - match: { aggregations.tsids.buckets.2.doc_count: 2 } -# - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} -# -# - match: { aggregations.tsids.buckets.3.key: "NCLYECP-GoaIfjk0RBfdlg3gKelzeRDHR3kQx0eLDvTuOYoRw6Wapg_P07YIhdV3NFJDjjE" } -# - match: { aggregations.tsids.buckets.3.doc_count: 2 } -# - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.65, error: 0.01 }} +--- +flattened dimension: + - skip: + features: close_to + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 + + + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + routing_path: [uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + uid: + type: keyword + time_series_dimension: true + deployment: + type: flattened + time_series_dimensions: [ "build.tag", "version.major", "version.minor", "version.patch" ] + + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.2, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.0, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.4, "deployment": { "build": { "tag": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.3, "deployment": { "build": { "tag": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.6, "deployment": { "build": { "tag": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "tag": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.8, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' + + - is_false: errors + - do: + search: + index: test + body: + size: 0 + aggs: + tsids: + terms: + field: _tsid + order: + _key: asc + aggs: + voltage: + avg: + field: voltage + + - match: { hits.total.value: 8} + - length: { aggregations.tsids.buckets: 4} + + - match: { aggregations.tsids.buckets.0.key: "NCLYECP-GoaIfjk0RBfdlg0oaZ29eRDHR3kQx0fjCuTKddqA4R9ytcYdqdbcoJ8VBuvqFtQ" } + - match: { aggregations.tsids.buckets.0.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.35, error: 0.01 }} + + - match: { aggregations.tsids.buckets.1.key: "NCLYECP-GoaIfjk0RBfdlg2rnf7qeRDHR3kQx0eLDvTuOYoRw4EeUIKK1KGFPmxqCWQyQJE" } + - match: { aggregations.tsids.buckets.1.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} + + - match: { aggregations.tsids.buckets.2.key: "NCLYECP-GoaIfjk0RBfdlg3P1l1UeRDHR3kQx0fjCuTKddqA4ai7CiSuCxA-PhBdLYOXkVY" } + - match: { aggregations.tsids.buckets.2.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} + + - match: { aggregations.tsids.buckets.3.key: "NCLYECP-GoaIfjk0RBfdlg3gKelzeRDHR3kQx0eLDvTuOYoRw6Wapg_P07YIhdV3NFJDjjE" } + - match: { aggregations.tsids.buckets.3.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.65, error: 0.01 }} --- flattened empty dimension: @@ -336,97 +336,97 @@ flattened field missing routing path field: - match: { aggregations.tsids.buckets.5.doc_count: 1 } - close_to: { aggregations.tsids.buckets.5.voltage.value: { value: 6.60, error: 0.01 }} -#--- -#flattened field misspelled routing path field: -# - skip: -# features: close_to -# version: " - 8.11.99" -# reason: _tsid hasing introduced in 8.12 -# -# - do: -# indices.create: -# index: test -# body: -# settings: -# index: -# mode: time_series -# # NOTE: 'reigion' here is misspelled on purpose -# routing_path: [deployment.reigion, deployment.build.tag] -# time_series: -# start_time: 2021-04-28T00:00:00Z -# end_time: 2021-04-29T00:00:00Z -# mappings: -# properties: -# "@timestamp": -# type: date -# uid: -# type: keyword -# deployment: -# type: flattened -# # NOTE: 'reigion' here is misspelled on purpose -# time_series_dimensions: [ reigion, build.tag ] -# -# - do: -# bulk: -# refresh: true -# index: test -# body: -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.2, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.0, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.4, "deployment": { "build": { "tag": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.3, "deployment": { "build": { "sha": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.6, "deployment": { "build": { "tag": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "sha": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' -# - '{"index": {}}' -# - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.8, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' -# -# - is_true: errors -# -# - match: { items.3.index.error.reason: "Error extracting routing: source didn't contain any routing fields" } -# - match: { items.5.index.error.reason: "Error extracting routing: source didn't contain any routing fields" } -# -# - do: -# search: -# index: test -# body: -# size: 0 -# aggs: -# tsids: -# terms: -# field: _tsid -# order: -# _key: asc -# aggs: -# voltage: -# avg: -# field: voltage -# -# - match: { hits.total.value: 6 } -# - length: { aggregations.tsids.buckets: 4 } -# -# - match: { aggregations.tsids.buckets.0.key: "JBs0-JZ2yoAg-Lrw35Mu3ysoaZ29egRdyNeHXPSPghDVzguaRg" } -# - match: { aggregations.tsids.buckets.0.doc_count: 1 } -# - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} -# -# - match: { aggregations.tsids.buckets.1.key: "JBs0-JZ2yoAg-Lrw35Mu3yurnf7qs9-VXFZ6jjZCbl_iiXSs7Q" } -# - match: { aggregations.tsids.buckets.1.doc_count: 2 } -# - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} -# -# - match: { aggregations.tsids.buckets.2.key: "JBs0-JZ2yoAg-Lrw35Mu3yvP1l1UlmlXEQVNXrHpUvpn7by0jA" } -# - match: { aggregations.tsids.buckets.2.doc_count: 2 } -# - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} -# -# - match: { aggregations.tsids.buckets.3.key: "JBs0-JZ2yoAg-Lrw35Mu3yvgKelz9WSJqzeYh7aza_7yxDXMZA" } -# - match: { aggregations.tsids.buckets.3.doc_count: 1 } -# - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} +--- +flattened field misspelled routing path field: + - skip: + features: close_to + version: " - 8.11.99" + reason: _tsid hasing introduced in 8.12 + + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + # NOTE: 'reigion' here is misspelled on purpose + routing_path: [deployment.reigion, deployment.build.tag] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + uid: + type: keyword + deployment: + type: flattened + # NOTE: 'reigion' here is misspelled on purpose + time_series_dimensions: [ reigion, build.tag ] + + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.2, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.0, "deployment": { "build": { "tag": "1516op6778", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.4, "deployment": { "build": { "tag": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "947e4ced-1786-4e53-9e0c-5c447e959507", "voltage": 7.3, "deployment": { "build": { "sha": "1516op6885", "branch": "release-8.8" }, "region": "eu-west-1", "version": { "major": 8, "minor": 8, "patch": 0 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:24.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.6, "deployment": { "build": { "tag": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:34.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "sha": "16w3xaca09", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:44.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.7, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.8, "deployment": { "build": { "tag": "16w3xacq34", "branch": "release-8.8" }, "region": "eu-west-2", "version": { "major": 8, "minor": 8, "patch": 1 }}}' + + - is_true: errors + + - match: { items.3.index.error.reason: "Error extracting routing: source didn't contain any routing fields" } + - match: { items.5.index.error.reason: "Error extracting routing: source didn't contain any routing fields" } + + - do: + search: + index: test + body: + size: 0 + aggs: + tsids: + terms: + field: _tsid + order: + _key: asc + aggs: + voltage: + avg: + field: voltage + + - match: { hits.total.value: 6 } + - length: { aggregations.tsids.buckets: 4 } + + - match: { aggregations.tsids.buckets.0.key: "JBs0-JZ2yoAg-Lrw35Mu3ysoaZ29egRdyNeHXPSPghDVzguaRg" } + - match: { aggregations.tsids.buckets.0.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.0.voltage.value: { value: 7.40, error: 0.01 }} + + - match: { aggregations.tsids.buckets.1.key: "JBs0-JZ2yoAg-Lrw35Mu3yurnf7qs9-VXFZ6jjZCbl_iiXSs7Q" } + - match: { aggregations.tsids.buckets.1.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.1.voltage.value: { value: 6.75, error: 0.01 }} + + - match: { aggregations.tsids.buckets.2.key: "JBs0-JZ2yoAg-Lrw35Mu3yvP1l1UlmlXEQVNXrHpUvpn7by0jA" } + - match: { aggregations.tsids.buckets.2.doc_count: 2 } + - close_to: { aggregations.tsids.buckets.2.voltage.value: { value: 7.10, error: 0.01 }} + + - match: { aggregations.tsids.buckets.3.key: "JBs0-JZ2yoAg-Lrw35Mu3yvgKelz9WSJqzeYh7aza_7yxDXMZA" } + - match: { aggregations.tsids.buckets.3.doc_count: 1 } + - close_to: { aggregations.tsids.buckets.3.voltage.value: { value: 6.60, error: 0.01 }} --- long dimension: diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 37cb8e638831b..ea5de46bbc462 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -699,7 +699,7 @@ public Object format(BytesRef value) { // NOTE: if the tsid is a map of dimension key/value pairs (as it was before introducing // tsid hashing) we just decode the map and return it. return TimeSeriesIdFieldMapper.decodeTsidAsMap(value); - } catch (IllegalArgumentException iaex) { + } catch (Throwable t) { // NOTE: otherwise the _tsid field is just a hash and we can't decode it return TimeSeriesIdFieldMapper.encodeTsid(value); } From 126f87722a07cd2e4a0b27b4b6e500e95ada307c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 12 Dec 2023 12:32:00 +0100 Subject: [PATCH 087/125] fix: sort dimensions by name --- .../aggregations/bucket/TimeSeriesAggregationsIT.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java index 2050ce20b1aee..9950ede4ef7bc 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java @@ -41,11 +41,13 @@ import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -146,7 +148,7 @@ public void setupSuiteScopeCluster() throws Exception { for (int i = 0; i < numberOfDocs; i++) { XContentBuilder docSource = XContentFactory.jsonBuilder(); docSource.startObject(); - Map key = new HashMap<>(); + Map key = new TreeMap<>(Comparator.naturalOrder()); for (int d = 0; d < numberOfDimensions; d++) { String dim = randomFrom(dimensions[d]); docSource.field("dim_" + d, dim); From f3bfc5089d8bb50ab5e16e6b84ce508b68f62712 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 12 Dec 2023 12:32:27 +0100 Subject: [PATCH 088/125] fix: temporarily fix the number of indices to 1 --- .../aggregations/bucket/TimeSeriesAggregationsIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java index 9950ede4ef7bc..0b8aef7366ce3 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsIT.java @@ -78,7 +78,7 @@ public class TimeSeriesAggregationsIT extends AggregationIntegTestCase { @Override public void setupSuiteScopeCluster() throws Exception { - int numberOfIndices = randomIntBetween(1, 3); + int numberOfIndices = randomIntBetween(1, 1); numberOfDimensions = randomIntBetween(1, 5); numberOfMetrics = randomIntBetween(1, 10); String[] routingKeys = randomSubsetOf( From 3ed353430c7daa5fcfcb4ea855c9046a44ff182d Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 12 Dec 2023 12:53:10 +0100 Subject: [PATCH 089/125] fix: update yaml tests skip version --- .../test/data_stream/150_tsdb.yml | 8 ++--- .../rest-api-spec/test/delete/70_tsdb.yml | 4 +-- .../rest-api-spec/test/tsdb/100_composite.yml | 8 ++--- .../test/tsdb/140_routing_path.yml | 8 ++--- .../test/tsdb/25_id_generation.yml | 32 +++++++++---------- .../rest-api-spec/test/tsdb/30_snapshot.yml | 4 +-- .../rest-api-spec/test/tsdb/40_search.yml | 16 +++++----- .../rest-api-spec/test/tsdb/50_alias.yml | 8 ++--- .../test/tsdb/60_add_dimensions.yml | 20 ++++++------ .../test/tsdb/70_dimension_types.yml | 28 ++++++++-------- .../test/tsdb/80_index_resize.yml | 12 +++---- .../test/downsample/10_basic.yml | 24 +++++++------- .../test/downsample/40_runtime_fields.yml | 12 +++---- .../test/downsample/60_settings.yml | 4 +-- .../test/downsample/10_basic.yml | 4 +-- .../rest-api-spec/test/analytics/100_tsdb.yml | 5 +-- 16 files changed, 99 insertions(+), 98 deletions(-) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 442ba800200a2..42dc83b6e5bef 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -133,8 +133,8 @@ created the data stream: --- fetch the tsid: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: search: @@ -152,8 +152,8 @@ fetch the tsid: --- "aggregate the tsid": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml index ce61e0a0711d0..207a23b8f212b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml @@ -1,8 +1,8 @@ --- "basic tsdb delete": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml index 3da8d8fa341c7..0f963c11632d3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml @@ -64,8 +64,8 @@ setup: --- composite aggregation on tsid: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: search: @@ -114,8 +114,8 @@ composite aggregation on tsid: --- composite aggregation on tsid with after: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 275bd446d3996..6f0f846807279 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -2,8 +2,8 @@ missing routing path field: - skip: features: close_to - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: @@ -122,8 +122,8 @@ missing dimension on routing path field: multi-value routing path field: - skip: features: close_to - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 66430e9c4b2e8..4069f601fe4ef 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 8.1.99,8.7.00 - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.1.99,8.7.00 - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: @@ -136,8 +136,8 @@ generates a consistent id: --- index a new document on top of an old one: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: search: @@ -258,8 +258,8 @@ create operation on top of old document fails: --- create operation on top of old document fails over bulk: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: bulk: @@ -273,8 +273,8 @@ create operation on top of old document fails over bulk: --- ids query: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: search: @@ -304,8 +304,8 @@ ids query: --- get: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: search: @@ -363,8 +363,8 @@ get with routing: --- delete: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: delete: @@ -441,8 +441,8 @@ delete over _bulk: --- routing_path matches deep object: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: @@ -486,8 +486,8 @@ routing_path matches deep object: --- routing_path matches object: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 142e17e4f665c..1e6dc803e001e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: "8.7.00 - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: "8.7.00 - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: snapshot.create_repository: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index beb36631c6487..35fcb22118fbc 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 8.1.99,8.7.00 - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.1.99,8.7.00 - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: @@ -171,8 +171,8 @@ fetch a tag: --- "fetch the tsid": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: search: @@ -266,8 +266,8 @@ aggregate a tag: --- "aggregate the tsid": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: search: @@ -309,8 +309,8 @@ aggregate a tag: --- sort by tsid: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index c1b304cb1c155..45ba4201ae2e0 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -64,8 +64,8 @@ setup: --- search an alias: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.put_alias: @@ -93,8 +93,8 @@ search an alias: --- index into alias: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.put_alias: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 4f9089533ba8e..2354e14dd51cc 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -1,8 +1,8 @@ --- add dimensions with put_mapping: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: @@ -54,8 +54,8 @@ add dimensions with put_mapping: --- add dimensions to no dims with dynamic_template over index: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: @@ -102,8 +102,8 @@ add dimensions to no dims with dynamic_template over index: --- add dimensions to no dims with dynamic_template over bulk: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: @@ -150,8 +150,8 @@ add dimensions to no dims with dynamic_template over bulk: --- add dimensions to some dims with dynamic_template over index: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: @@ -202,8 +202,8 @@ add dimensions to some dims with dynamic_template over index: --- add dimensions to some dims with dynamic_template over bulk: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 74eb8ea9ad4fa..2975fd7b40f5b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -2,8 +2,8 @@ keyword dimension: - skip: features: close_to - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: @@ -76,8 +76,8 @@ keyword dimension: flattened dimension: - skip: features: close_to - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: @@ -164,8 +164,8 @@ flattened dimension: flattened empty dimension: - skip: features: close_to - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: @@ -245,8 +245,8 @@ flattened empty dimension: flattened field missing routing path field: - skip: features: close_to - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: @@ -340,8 +340,8 @@ flattened field missing routing path field: flattened field misspelled routing path field: - skip: features: close_to - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: @@ -432,8 +432,8 @@ flattened field misspelled routing path field: long dimension: - skip: features: close_to - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: @@ -507,8 +507,8 @@ long dimension: ip dimension: - skip: features: close_to - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index 4e3bc2510e55c..db4536fdd94be 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: " - 8.1.99,8.7.00 - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.1.99,8.7.00 - 8.12.99" + reason: _tsid hasing introduced in 8.13 features: "arbitrary_key" # Force allocating all shards to a single node so that we can shrink later. @@ -102,8 +102,8 @@ split: --- shrink: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.shrink: @@ -129,8 +129,8 @@ shrink: --- clone: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.clone: diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index 7a6e364ebba6a..ad08030904983 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -289,8 +289,8 @@ setup: --- "Downsample index": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.downsample: @@ -678,8 +678,8 @@ setup: --- "Downsample a downsampled index": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.downsample: @@ -817,8 +817,8 @@ setup: --- "Downsample histogram as label": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.downsample: @@ -1229,8 +1229,8 @@ setup: --- "Downsample object field": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.downsample: @@ -1297,8 +1297,8 @@ setup: --- "Downsample empty and missing labels": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.downsample: @@ -1358,8 +1358,8 @@ setup: --- "Downsample label with ignore_above": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml index 0cd94cc56eb77..3f27ec6f28a5e 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml @@ -1,8 +1,8 @@ --- "Runtime fields accessing metric fields in downsample target index": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 features: close_to - do: @@ -163,8 +163,8 @@ --- "Runtime field accessing dimension fields in downsample target index": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: @@ -284,8 +284,8 @@ --- "Runtime field accessing label fields in downsample target index": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.create: diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml index 74c2020d9a0d6..e272681f6d29c 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml @@ -93,8 +93,8 @@ --- "Downsample datastream with tier preference": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 features: allowed_warnings - do: diff --git a/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index 6605aac85a017..0084030255761 100644 --- a/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -293,8 +293,8 @@ setup: --- "Downsample index": - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 - do: indices.downsample: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml index 2c99a691f127b..cf618e183457a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml @@ -63,8 +63,9 @@ setup: --- aggretate multi_terms: - skip: - version: " - 8.11.99" - reason: _tsid hasing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 + - do: search: size: 0 From f8af828e58cbd9f94a353337a343bcf5aad69088 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 12 Dec 2023 15:32:33 +0100 Subject: [PATCH 090/125] fix: update a couple of tests to version 8.13 --- .../elasticsearch/upgrades/IndexingIT.java | 16 +++--- .../composite/CompositeAggregatorTests.java | 50 ++++++------------- 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java index 438e519eb5c77..f2fbd506c75f3 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java @@ -275,8 +275,8 @@ public void testTsdb() throws IOException { oldClusterVersion, EXPECTED_TSDB_TSIDS_NODES_1, closeTo(217.45, 0.005), - closeTo(2391.95, 0.005), - closeTo(-217.45, 0.005) + closeTo(-217.45, 0.005), + closeTo(2391.95, 0.005) ); } else { assertTsdbAgg( @@ -298,9 +298,9 @@ public void testTsdb() throws IOException { oldClusterVersion, EXPECTED_TSDB_TSIDS_NODES_2, closeTo(218.95, 0.5), - closeTo(21895.0, 0.005), + closeTo(-218.95, 0.005), closeTo(2408.45, 0.005), - closeTo(-218.95, 0.005) + closeTo(21895.0, 0.005) ); } else { assertTsdbAgg( @@ -324,10 +324,10 @@ public void testTsdb() throws IOException { oldClusterVersion, EXPECTED_TSDB_TSIDS_NODES_3, closeTo(220.45, 0.005), - closeTo(-11022.5, 0.5), - closeTo(22045, 0.5), + closeTo(-220.45, 0.005), closeTo(2424.95, 0.005), - closeTo(-220.45, 0.005) + closeTo(22045, 0.5), + closeTo(-11022.5, 0.5) ); } else { assertTsdbAgg( @@ -383,7 +383,7 @@ private void tsdbBulk(StringBuilder bulk, String dim, long timeStart, long timeE private void assertTsdbAgg(final Version oldClusterVersion, final List expectedTsids, final Matcher... expected) throws IOException { - boolean onOrAfterTsidHashingVersion = oldClusterVersion.onOrAfter(Version.V_8_12_0); + boolean onOrAfterTsidHashingVersion = oldClusterVersion.onOrAfter(Version.V_8_13_0); Request request = new Request("POST", "/tsdb/_search"); request.addParameter("size", "0"); XContentBuilder body = JsonXContent.contentBuilder().startObject(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index dedcfcd1eff27..5859c1c162c43 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -2877,15 +2877,15 @@ public void testWithTsid() throws Exception { () -> new CompositeAggregationBuilder("name", Collections.singletonList(new TermsValuesSourceBuilder("tsid").field("_tsid"))), (InternalComposite result) -> { assertEquals(4, result.getBuckets().size()); - assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA}", result.afterKey().toString()); + assertEquals("{tsid={dim1=foo, dim2=200}}", result.afterKey().toString()); - assertEquals("{tsid=AgRkaW0xcwNiYXIEZGltMmwAAAAAAAAAZA}", result.getBuckets().get(0).getKeyAsString()); + assertEquals("{tsid={dim1=bar, dim2=100}}", result.getBuckets().get(0).getKeyAsString()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); - assertEquals("{tsid=AgRkaW0xcwNiYXIEZGltMmwAAAAAAAAAyA}", result.getBuckets().get(1).getKeyAsString()); + assertEquals("{tsid={dim1=bar, dim2=200}}", result.getBuckets().get(1).getKeyAsString()); assertEquals(1L, result.getBuckets().get(1).getDocCount()); - assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAZA}", result.getBuckets().get(2).getKeyAsString()); + assertEquals("{tsid={dim1=foo, dim2=100}}", result.getBuckets().get(2).getKeyAsString()); assertEquals(2L, result.getBuckets().get(2).getDocCount()); - assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA}", result.getBuckets().get(3).getKeyAsString()); + assertEquals("{tsid={dim1=foo, dim2=200}}", result.getBuckets().get(3).getKeyAsString()); assertEquals(1L, result.getBuckets().get(3).getDocCount()); } ); @@ -2897,10 +2897,10 @@ public void testWithTsid() throws Exception { ); }, (InternalComposite result) -> { assertEquals(2, result.getBuckets().size()); - assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA}", result.afterKey().toString()); - assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAZA}", result.getBuckets().get(0).getKeyAsString()); + assertEquals("{tsid={dim1=foo, dim2=200}}", result.afterKey().toString()); + assertEquals("{tsid={dim1=foo, dim2=100}}", result.getBuckets().get(0).getKeyAsString()); assertEquals(2L, result.getBuckets().get(0).getDocCount()); - assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA}", result.getBuckets().get(1).getKeyAsString()); + assertEquals("{tsid={dim1=foo, dim2=200}}", result.getBuckets().get(1).getKeyAsString()); assertEquals(1L, result.getBuckets().get(1).getDocCount()); }); } @@ -2929,27 +2929,15 @@ public void testWithTsidAndDateHistogram() throws IOException { ), (InternalComposite result) -> { assertEquals(4, result.getBuckets().size()); - assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA, date_histo=1634688000000}", result.afterKey().toString()); + assertEquals("{tsid={dim1=foo, dim2=200}, date_histo=1634688000000}", result.afterKey().toString()); - assertEquals( - "{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAZA, date_histo=1632096000000}", - result.getBuckets().get(0).getKeyAsString() - ); + assertEquals("{tsid={dim1=foo, dim2=100}, date_histo=1632096000000}", result.getBuckets().get(0).getKeyAsString()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); - assertEquals( - "{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAZA, date_histo=1634688000000}", - result.getBuckets().get(1).getKeyAsString() - ); + assertEquals("{tsid={dim1=foo, dim2=100}, date_histo=1634688000000}", result.getBuckets().get(1).getKeyAsString()); assertEquals(1L, result.getBuckets().get(1).getDocCount()); - assertEquals( - "{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA, date_histo=1632096000000}", - result.getBuckets().get(2).getKeyAsString() - ); + assertEquals("{tsid={dim1=foo, dim2=200}, date_histo=1632096000000}", result.getBuckets().get(2).getKeyAsString()); assertEquals(1L, result.getBuckets().get(2).getDocCount()); - assertEquals( - "{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA, date_histo=1634688000000}", - result.getBuckets().get(3).getKeyAsString() - ); + assertEquals("{tsid={dim1=foo, dim2=200}, date_histo=1634688000000}", result.getBuckets().get(3).getKeyAsString()); assertEquals(2L, result.getBuckets().get(3).getDocCount()); } ); @@ -2966,16 +2954,10 @@ public void testWithTsidAndDateHistogram() throws IOException { ).aggregateAfter(createAfterKey("tsid", createTsid(Map.of("dim1", "foo", "dim2", 100)), "date_histo", 1634688000000L)), (InternalComposite result) -> { assertEquals(2, result.getBuckets().size()); - assertEquals("{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA, date_histo=1634688000000}", result.afterKey().toString()); - assertEquals( - "{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA, date_histo=1632096000000}", - result.getBuckets().get(0).getKeyAsString() - ); + assertEquals("{tsid={dim1=foo, dim2=200}, date_histo=1634688000000}", result.afterKey().toString()); + assertEquals("{tsid={dim1=foo, dim2=200}, date_histo=1632096000000}", result.getBuckets().get(0).getKeyAsString()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); - assertEquals( - "{tsid=AgRkaW0xcwNmb28EZGltMmwAAAAAAAAAyA, date_histo=1634688000000}", - result.getBuckets().get(1).getKeyAsString() - ); + assertEquals("{tsid={dim1=foo, dim2=200}, date_histo=1634688000000}", result.getBuckets().get(1).getKeyAsString()); assertEquals(2L, result.getBuckets().get(1).getDocCount()); } ); From 926ba0c4094d1a5aa272b94bc84044ebd7adae4e Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 12 Dec 2023 15:53:58 +0100 Subject: [PATCH 091/125] fix: another yaml test after switch to 8.13 --- .../test/aggregations/time_series.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index e0069782bb2b2..edcb1eeb8c883 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -107,8 +107,8 @@ setup: --- "Size test": - skip: - version: " - 8.11.99" - reason: _tsid hashing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: search: @@ -126,7 +126,7 @@ setup: size: 1 - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: { "key": "baz" } } + - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } - do: search: @@ -144,9 +144,9 @@ setup: size: 3 - length: { aggregations.ts.buckets: 3 } - - match: { aggregations.ts.buckets.0.key: { "key": "baz" } } - - match: { aggregations.ts.buckets.1.key: { "key": "foo" } } - - match: { aggregations.ts.buckets.2.key: { "key": "bar" } } + - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } + - match: { aggregations.ts.buckets.1.key: { "key": "baz" } } + - match: { aggregations.ts.buckets.2.key: { "key": "foo" } } --- "Score test filter some": @@ -311,8 +311,8 @@ setup: --- "Number for keyword routing field": - skip: - version: " - 8.11.99" - reason: _tsid hashing introduced in 8.12 + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 - do: bulk: From 6803f288b69aa025be9e97b3e33d46bb0206728a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 13 Dec 2023 11:55:43 +0100 Subject: [PATCH 092/125] fix: _tsid sort vs _tsid hash sort bucket creation --- .../timeseries/TimeSeriesAggregator.java | 7 ++ .../test/aggregations/time_series.yml | 111 +++++++++++++++++- 2 files changed, 114 insertions(+), 4 deletions(-) diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java index f0774baf4c8c6..9cd7f7a86e532 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.SortedMap; @@ -85,6 +86,12 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I break; } } + // NOTE: after introducing _tsid hashing time series are sorted by (_tsid hash, @timestamp) instead of (_tsid, timestamp). + // _tsid hash and _tsid might sort differently, and out of order data might result in incorrect buckets due to _tsid value + // changes not matching _tsid hash changes. Changes in _tsid hash are handled creating a new bucket as a result of making + // the assumption that sorting data results in new buckets whenever there is a change in _tsid hash. This is no true anymore + // because we collect data sorted on (_tsid hash, timestamp) but build aggregation results sorted by (_tsid, timestamp). + buckets.sort(Comparator.comparing(bucket -> bucket.key)); allBucketsPerOrd[ordIdx] = buckets.toArray(new InternalTimeSeries.InternalBucket[0]); } buildSubAggsForAllBuckets(allBucketsPerOrd, b -> b.bucketOrd, (b, a) -> b.aggregations = a); diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index edcb1eeb8c883..35f7c46124675 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -126,7 +126,7 @@ setup: size: 1 - length: { aggregations.ts.buckets: 1 } - - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } + - match: { aggregations.ts.buckets.0.key: { "key": "baz" } } - do: search: @@ -144,9 +144,9 @@ setup: size: 3 - length: { aggregations.ts.buckets: 3 } - - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } - - match: { aggregations.ts.buckets.1.key: { "key": "baz" } } - - match: { aggregations.ts.buckets.2.key: { "key": "foo" } } + - match: { aggregations.ts.buckets.0.key: { "key": "baz" } } + - match: { aggregations.ts.buckets.1.key: { "key": "foo" } } + - match: { aggregations.ts.buckets.2.key: { "key": "bar" } } --- "Score test filter some": @@ -345,3 +345,106 @@ setup: - match: { aggregations.ts.buckets.0.doc_count: 1 } - match: { aggregations.ts.buckets.1.key: { "key": "10" } } - match: { aggregations.ts.buckets.1.doc_count: 1 } + +--- +"Multiple indices _tsid vs _tsid hash sorting": +# sort(_tsid, timestamp) != sort(_tsid hash, timestamp) might result in incorrect buckets in the aggregation result. +# Here dimension values are crafted in such a way that sorting on _tsid and sorting on _tsid hash results in different +# collection and reduction order. Note that changing the hashing algorithm might require selecting proper values +# for dimensions fields such that sort(_tsid, timestamp) != sort(_tsid hash, timestamp). + + - do: + indices.create: + index: test-1 + body: + settings: + mode: time_series + routing_path: [ key ] + time_series: + start_time: "2021-04-01T00:00:00Z" + end_time: "2021-04-30T23:59:59Z" + number_of_shards: 1 + mappings: + properties: + key: + type: keyword + time_series_dimension: true + "@timestamp": + type: date + gauge: + type: double + time_series_metric: "gauge" + + - do: + indices.create: + index: test-2 + body: + settings: + mode: time_series + routing_path: [ key ] + time_series: + start_time: "2021-05-01T00:00:00Z" + end_time: "2022-05-31T23:59:59Z" + number_of_shards: 1 + mappings: + properties: + key: + type: keyword + time_series_dimension: true + "@timestamp": + type: date + gauge: + type: double + time_series_metric: "gauge" + + - do: + bulk: + index: test-1 + refresh: true + body: + - '{ "index": {} }' + - '{ "key": "bar", "gauge": 1, "@timestamp": "2021-04-01T01:00:11Z" }' + - '{ "index": {} }' + - '{ "key": "bar", "gauge": 2, "@timestamp": "2021-04-01T02:00:12Z" }' + - '{ "index": {} }' + - '{ "key": "foo", "gauge": 3, "@timestamp": "2021-04-01T03:00:13Z" }' + + - is_false: errors + + - do: + bulk: + index: test-2 + refresh: true + body: + - '{ "index": {} }' + - '{ "key": "bar", "gauge": 10, "@timestamp": "2021-05-01T01:00:31Z" }' + - '{ "index": {} }' + - '{ "key": "foo", "gauge": 20, "@timestamp": "2021-05-01T02:00:32Z" }' + - '{ "index": {} }' + - '{ "key": "bar", "gauge": 30, "@timestamp": "2021-05-01T03:00:33Z" }' + - '{ "index": {} }' + - '{ "key": "foo", "gauge": 40, "@timestamp": "2021-05-01T04:00:34Z" }' + + - is_false: errors + + - do: + search: + index: test-1,test-2 + body: + fields: + - field: "_tsid" + - field: "@timestamp" + size: 10 + aggs: + ts: + time_series: + keyed: false + + - match: { hits.total.value: 7 } + - length: { aggregations: 1 } + + - length: { aggregations.ts.buckets: 2 } + - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } + - match: { aggregations.ts.buckets.0.doc_count: 4 } + - match: { aggregations.ts.buckets.1.key: { "key": "foo" } } + - match: { aggregations.ts.buckets.1.doc_count: 3 } From f712b180dd4215f2ec26f7bd566a7968b2eadfcb Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 13 Dec 2023 13:03:19 +0100 Subject: [PATCH 093/125] fix: bucket ordering --- .../test/aggregations/time_series.yml | 10 +++++----- .../elasticsearch/upgrades/IndexingIT.java | 20 +++++++++---------- .../aggregations/GeoLineAggregatorTests.java | 12 ++++------- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 35f7c46124675..b6dd2d655ea40 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -144,9 +144,9 @@ setup: size: 3 - length: { aggregations.ts.buckets: 3 } - - match: { aggregations.ts.buckets.0.key: { "key": "baz" } } - - match: { aggregations.ts.buckets.1.key: { "key": "foo" } } - - match: { aggregations.ts.buckets.2.key: { "key": "bar" } } + - match: { aggregations.ts.buckets.0.key: { "key": "bar" } } + - match: { aggregations.ts.buckets.1.key: { "key": "baz" } } + - match: { aggregations.ts.buckets.2.key: { "key": "foo" } } --- "Score test filter some": @@ -341,9 +341,9 @@ setup: - match: { hits.total.value: 2 } - length: { aggregations: 1 } - - match: { aggregations.ts.buckets.0.key: { "key": "11" } } + - match: { aggregations.ts.buckets.0.key: { "key": "10" } } - match: { aggregations.ts.buckets.0.doc_count: 1 } - - match: { aggregations.ts.buckets.1.key: { "key": "10" } } + - match: { aggregations.ts.buckets.1.key: { "key": "11" } } - match: { aggregations.ts.buckets.1.doc_count: 1 } --- diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java index f2fbd506c75f3..113fa1cd04b10 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexingIT.java @@ -270,13 +270,13 @@ public void testTsdb() throws IOException { tsdbBulk(bulk, TSDB_DIMS.get(1), TSDB_TIMES[1], TSDB_TIMES[2], -0.1); tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[0], TSDB_TIMES[2], 1.1); bulk("tsdb", bulk.toString()); - if (oldClusterVersion.onOrAfter(Version.V_8_12_0)) { + if (oldClusterVersion.onOrAfter(Version.V_8_13_0)) { assertTsdbAgg( oldClusterVersion, EXPECTED_TSDB_TSIDS_NODES_1, closeTo(217.45, 0.005), - closeTo(-217.45, 0.005), - closeTo(2391.95, 0.005) + closeTo(2391.95, 0.005), + closeTo(-217.45, 0.005) ); } else { assertTsdbAgg( @@ -293,14 +293,14 @@ public void testTsdb() throws IOException { tsdbBulk(bulk, TSDB_DIMS.get(2), TSDB_TIMES[2], TSDB_TIMES[3], 1.1); tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[0], TSDB_TIMES[3], 10); bulk("tsdb", bulk.toString()); - if (oldClusterVersion.onOrAfter(Version.V_8_12_0)) { + if (oldClusterVersion.onOrAfter(Version.V_8_13_0)) { assertTsdbAgg( oldClusterVersion, EXPECTED_TSDB_TSIDS_NODES_2, closeTo(218.95, 0.5), - closeTo(-218.95, 0.005), + closeTo(21895.0, 0.005), closeTo(2408.45, 0.005), - closeTo(21895.0, 0.005) + closeTo(-218.95, 0.005) ); } else { assertTsdbAgg( @@ -319,15 +319,15 @@ public void testTsdb() throws IOException { tsdbBulk(bulk, TSDB_DIMS.get(3), TSDB_TIMES[3], TSDB_TIMES[4], 10); tsdbBulk(bulk, TSDB_DIMS.get(4), TSDB_TIMES[0], TSDB_TIMES[4], -5); bulk("tsdb", bulk.toString()); - if (oldClusterVersion.onOrAfter(Version.V_8_12_0)) { + if (oldClusterVersion.onOrAfter(Version.V_8_13_0)) { assertTsdbAgg( oldClusterVersion, EXPECTED_TSDB_TSIDS_NODES_3, closeTo(220.45, 0.005), - closeTo(-220.45, 0.005), - closeTo(2424.95, 0.005), + closeTo(-11022.5, 0.5), closeTo(22045, 0.5), - closeTo(-11022.5, 0.5) + closeTo(2424.95, 0.005), + closeTo(-220.45, 0.005) ); } else { assertTsdbAgg( diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index 194a53944d037..0b76e786b26be 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -360,8 +360,7 @@ public void testGeoLine_TSDB() throws IOException { assertThat("Number of time-series buckets", tsx.ts.getBuckets().size(), equalTo(g)); for (int i = 0; i < tsx.groups.length; i++) { InternalGeoLine geoLine = tsx.ts.getBuckets().get(i).getAggregations().get("track"); - int index = testConfig.useTimeSeriesAggregation ? tsx.groups.length - i - 1 : i; - assertGeoLine(sortOrder, tsx.groups[index], geoLine, tsx, true); + assertGeoLine(sortOrder, tsx.groups[i], geoLine, tsx, true); } }); } @@ -384,8 +383,7 @@ public void testGeoLine_Terms_TSDB() throws IOException { StringTerms terms = tsx.ts.getBuckets().get(i).getAggregations().get("groups"); assertThat("Number of terms buckets", terms.getBuckets().size(), equalTo(1)); InternalGeoLine geoLine = terms.getBuckets().get(0).getAggregations().get("track"); - int index = testConfig.useTimeSeriesAggregation ? tsx.groups.length - i - 1 : 0; - assertGeoLine(sortOrder, tsx.groups[index], geoLine, tsx, true); + assertGeoLine(sortOrder, tsx.groups[i], geoLine, tsx, true); } }); } @@ -405,8 +403,7 @@ public void testGeoLine_TSDB_simplified() throws IOException { assertThat("Number of time-series buckets", tsx.ts.getBuckets().size(), equalTo(g)); for (int i = 0; i < tsx.groups.length; i++) { InternalGeoLine geoLine = tsx.ts.getBuckets().get(i).getAggregations().get("track"); - int index = testConfig.useTimeSeriesAggregation ? tsx.groups.length - i - 1 : i; - assertGeoLine(sortOrder, tsx.groups[index], geoLine, tsx, false); + assertGeoLine(sortOrder, tsx.groups[i], geoLine, tsx, false); } }); } @@ -429,8 +426,7 @@ public void testGeoLine_Terms_TSDB_simplified() throws IOException { StringTerms terms = tsx.ts.getBuckets().get(i).getAggregations().get("groups"); assertThat("Number of terms buckets", terms.getBuckets().size(), equalTo(1)); InternalGeoLine geoLine = terms.getBuckets().get(0).getAggregations().get("track"); - int index = testConfig.useTimeSeriesAggregation ? tsx.groups.length - i - 1 : 0; - assertGeoLine(sortOrder, tsx.groups[index], geoLine, tsx, false); + assertGeoLine(sortOrder, tsx.groups[i], geoLine, tsx, false); } }); } From 1d8f99b5d4c6c2b6ba4ee7b21bbb23e5910efc5d Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 13 Dec 2023 14:25:57 +0100 Subject: [PATCH 094/125] fix: missing skip version for time series test --- .../resources/rest-api-spec/test/aggregations/time_series.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index b6dd2d655ea40..431ac4232fa09 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -353,6 +353,10 @@ setup: # collection and reduction order. Note that changing the hashing algorithm might require selecting proper values # for dimensions fields such that sort(_tsid, timestamp) != sort(_tsid hash, timestamp). + - skip: + version: " - 8.12.99" + reason: _tsid hashing introduced in 8.13 + - do: indices.create: index: test-1 From a638502fb4ee5f71dc69284712d9f4323ab4af84 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 13 Dec 2023 14:32:19 +0100 Subject: [PATCH 095/125] test: unnecessary fields and hits --- .../rest-api-spec/test/aggregations/time_series.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 431ac4232fa09..2cd2d5e027f68 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -435,10 +435,7 @@ setup: search: index: test-1,test-2 body: - fields: - - field: "_tsid" - - field: "@timestamp" - size: 10 + size: 0 aggs: ts: time_series: From 09ef74bbc81a5b172cc34cee69b7cc1ab5585515 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 19 Dec 2023 18:47:37 +0100 Subject: [PATCH 096/125] fix: allow empty dimension values --- .../test/downsample/10_basic.yml | 83 +++++++++++++++++++ .../downsample/DimensionFieldProducer.java | 2 +- 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index ad08030904983..a1b2bd87c8f43 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -1481,3 +1481,86 @@ setup: - match: { test-downsample-label-ignore-above.mappings.properties.k8s.properties.pod.properties.label.type: keyword } - match: { test-downsample-label-ignore-above.mappings.properties.k8s.properties.pod.properties.label.ignore_above: 3 } + +--- +"Downsample index with empty dimension": + - skip: + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 + + - do: + indices.create: + index: test-empty-dim + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + index: + mode: time_series + routing_path: [ k8s.pod.name ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + name: + type: keyword + time_series_dimension: true + empty: + type: keyword + time_series_dimension: true + gauge: + type: long + time_series_metric: gauge + - do: + bulk: + refresh: true + index: test-empty-dim + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "k8s": {"pod": {"name": "cat", "gauge": 10, "empty": "" }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:14.467Z", "k8s": {"pod": {"name": "cat", "gauge": 20 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:24.467Z", "k8s": {"pod": {"name": "cat", "gauge": 12, "empty": null }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:34.467Z", "k8s": {"pod": {"name": "cat", "gauge": 18 }}}' + + - do: + indices.put_settings: + index: test-empty-dim + body: + index.blocks.write: true + + - do: + indices.downsample: + index: test-empty-dim + target_index: test-empty-dim-downsample + body: > + { + "fixed_interval": "1h" + } + - is_true: acknowledged + + - do: + search: + index: test-empty-dim-downsample + body: + sort: [ "_tsid", "@timestamp" ] + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source._doc_count: 3 } + - match: { hits.hits.0._source.k8s.pod.name: cat } + - match: { hits.hits.0._source.k8s.pod.empty: null } + - match: { hits.hits.1._source._doc_count: 1 } + - match: { hits.hits.1._source.k8s.pod.name: cat } + - match: { hits.hits.1._source.k8s.pod.empty: "" } diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java index 54e3bed8d4b32..69493e6de442e 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DimensionFieldProducer.java @@ -70,7 +70,7 @@ public boolean isEmpty() { @Override public void collect(FormattedDocValues docValues, int docId) throws IOException { if (docValues.advanceExact(docId) == false) { - throw new IllegalArgumentException("Unable to collect dimension [" + this.dimension.name + "]"); + return; } int docValueCount = docValues.docValueCount(); for (int i = 0; i < docValueCount; i++) { From 3105725f5732ad6b01f2f2cf7248f229847f01b6 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 20 Dec 2023 16:04:41 +0100 Subject: [PATCH 097/125] fix: if statetment and additional yaml test --- .../index/mapper/TimeSeriesIdFieldMapper.java | 4 +- .../test/downsample/10_basic.yml | 83 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 88f6bbbb563e2..df26047a39732 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -233,7 +233,9 @@ public BytesReference buildTsidHash() throws IOException { // NOTE: concatenate all dimension value hashes up to a certain number of dimensions int tsidHashStartIndex = tsidHashIndex; for (final Dimension dimension : dimensions) { - if ((tsidHashIndex - tsidHashStartIndex) >= 4 * numberOfDimensions) break; + if ((tsidHashIndex - tsidHashStartIndex) >= 4 * numberOfDimensions) { + break; + } final BytesRef dimensionValueBytesRef = dimension.value.toBytesRef(); ByteUtils.writeIntLE( StringHelper.murmurhash3_x86_32( diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index a1b2bd87c8f43..a0d474063d294 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -1564,3 +1564,86 @@ setup: - match: { hits.hits.1._source._doc_count: 1 } - match: { hits.hits.1._source.k8s.pod.name: cat } - match: { hits.hits.1._source.k8s.pod.empty: "" } + +--- +"Downsample index with empty dimension on routing path": + - skip: + version: " - 8.12.99" + reason: _tsid hasing introduced in 8.13 + + - do: + indices.create: + index: test-empty-dim-routing + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + index: + mode: time_series + routing_path: [ k8s.pod.name, k8s.pod.empty ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + name: + type: keyword + time_series_dimension: true + empty: + type: keyword + time_series_dimension: true + gauge: + type: long + time_series_metric: gauge + - do: + bulk: + refresh: true + index: test-empty-dim-routing + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "k8s": {"pod": {"name": "cat", "gauge": 10, "empty": "" }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:14.467Z", "k8s": {"pod": {"name": "cat", "gauge": 20 }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:24.467Z", "k8s": {"pod": {"name": "cat", "gauge": 12, "empty": null }}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:34.467Z", "k8s": {"pod": {"name": "cat", "gauge": 18 }}}' + + - do: + indices.put_settings: + index: test-empty-dim-routing + body: + index.blocks.write: true + + - do: + indices.downsample: + index: test-empty-dim-routing + target_index: test-empty-dim-routing-downsample + body: > + { + "fixed_interval": "1h" + } + - is_true: acknowledged + + - do: + search: + index: test-empty-dim-routing-downsample + body: + sort: [ "_tsid", "@timestamp" ] + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source._doc_count: 3 } + - match: { hits.hits.0._source.k8s.pod.name: cat } + - match: { hits.hits.0._source.k8s.pod.empty: null } + - match: { hits.hits.1._source._doc_count: 1 } + - match: { hits.hits.1._source.k8s.pod.name: cat } + - match: { hits.hits.1._source.k8s.pod.empty: "" } From 70849340bf9f28ed0cb96a0f0f4f9ac712b24ca3 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 20 Dec 2023 16:34:12 +0100 Subject: [PATCH 098/125] fix: typo --- server/src/main/java/org/elasticsearch/index/IndexVersions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 4810a073de988..44b1ddfdc4e34 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -96,7 +96,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion UPGRADE_LUCENE_9_9_1 = def(8_500_008, Version.LUCENE_9_9_1); public static final IndexVersion ES_VERSION_8_13 = def(8_500_009, Version.LUCENE_9_9_1); public static final IndexVersion NEW_INDEXVERSION_FORMAT = def(8_501_00_0, Version.LUCENE_9_9_1); - public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_501_00)1, Version.LUCENE_9_9_1); + public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_501_00_1, Version.LUCENE_9_9_1); /* * STOP! READ THIS FIRST! No, really, From 9027df9538ca9c6b4461dcb502547f0d56f9be38 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 21 Dec 2023 12:07:58 +0100 Subject: [PATCH 099/125] fix: validate max number of dimensions for versions before 8.13 --- .../index/mapper/DocumentDimensions.java | 35 +++++++++++++------ .../index/mapper/IpFieldMapper.java | 2 +- .../index/mapper/KeywordFieldMapper.java | 2 +- .../index/mapper/MapperService.java | 2 +- .../index/mapper/NumberFieldMapper.java | 2 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 29 ++++++++++++--- .../flattened/FlattenedFieldParser.java | 2 +- .../unsignedlong/UnsignedLongFieldMapper.java | 2 +- 8 files changed, 54 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java index fafa7f7a9cb12..3cf90f2385525 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java @@ -32,17 +32,19 @@ static DocumentDimensions fromIndexSettings(IndexSettings indexSettings) { * value is already computed in some cases when we want to collect * dimensions, so we can save re-computing the UTF-8 encoding. */ - void addString(String fieldName, BytesRef utf8Value); + DocumentDimensions addString(String fieldName, BytesRef utf8Value); - default void addString(String fieldName, String value) { - addString(fieldName, new BytesRef(value)); + default DocumentDimensions addString(String fieldName, String value) { + return addString(fieldName, new BytesRef(value)); } - void addIp(String fieldName, InetAddress value); + DocumentDimensions addIp(String fieldName, InetAddress value); - void addLong(String fieldName, long value); + DocumentDimensions addLong(String fieldName, long value); - void addUnsignedLong(String fieldName, long value); + DocumentDimensions addUnsignedLong(String fieldName, long value); + + DocumentDimensions validate(IndexSettings settings); /** * Makes sure that each dimension only appears on time. @@ -51,29 +53,40 @@ class OnlySingleValueAllowed implements DocumentDimensions { private final Set names = new HashSet<>(); @Override - public void addString(String fieldName, BytesRef value) { + public DocumentDimensions addString(String fieldName, BytesRef value) { add(fieldName); + return this; } // Override to skip the UTF-8 conversion that happens in the default implementation @Override - public void addString(String fieldName, String value) { + public DocumentDimensions addString(String fieldName, String value) { add(fieldName); + return this; } @Override - public void addIp(String fieldName, InetAddress value) { + public DocumentDimensions addIp(String fieldName, InetAddress value) { add(fieldName); + return this; } @Override - public void addLong(String fieldName, long value) { + public DocumentDimensions addLong(String fieldName, long value) { add(fieldName); + return this; } @Override - public void addUnsignedLong(String fieldName, long value) { + public DocumentDimensions addUnsignedLong(String fieldName, long value) { add(fieldName); + return this; + } + + @Override + public DocumentDimensions validate(final IndexSettings settings) { + // DO NOTHING + return this; } private void add(String fieldName) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 56a50c2dee0aa..f35ac578a1981 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -555,7 +555,7 @@ private static InetAddress value(XContentParser parser, InetAddress nullValue) t private void indexValue(DocumentParserContext context, InetAddress address) { if (dimension) { - context.getDimensions().addIp(fieldType().name(), address); + context.getDimensions().addIp(fieldType().name(), address).validate(context.indexSettings()); } if (indexed) { Field field = new InetAddressPoint(fieldType().name(), address); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index fb4f0f7c91a9b..6e41e2850a863 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -915,7 +915,7 @@ private void indexValue(DocumentParserContext context, String value) { final BytesRef binaryValue = new BytesRef(value); if (fieldType().isDimension()) { - context.getDimensions().addString(fieldType().name(), binaryValue); + context.getDimensions().addString(fieldType().name(), binaryValue).validate(context.indexSettings()); } // If the UTF8 encoding of the field value is bigger than the max length 32766, Lucene fill fail the indexing request and, to diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 71f3f7c674780..cbf2dd872da2f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -116,7 +116,7 @@ public enum MergeReason { ); public static final Setting INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING = Setting.longSetting( "index.mapping.dimension_fields.limit", - 9999, + 21, 0, Property.Dynamic, Property.IndexScope diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 091e3c61764b0..cf39a6fd5832c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1872,7 +1872,7 @@ public Number value(XContentParser parser) throws IllegalArgumentException, IOEx */ public void indexValue(DocumentParserContext context, Number numericValue) { if (dimension && numericValue != null) { - context.getDimensions().addLong(fieldType().name(), numericValue.longValue()); + context.getDimensions().addLong(fieldType().name(), numericValue.longValue()).validate(context.indexSettings()); } fieldType().type.addFields(context.doc(), fieldType().name(), numericValue, indexed, hasDocValues, stored); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index df26047a39732..b4c2d61fb2630 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldData; @@ -273,7 +274,7 @@ private int writeHash128(final MurmurHash3.Hash128 hash128, byte[] buffer, int t } @Override - public void addString(String fieldName, BytesRef utf8Value) { + public DocumentDimensions addString(String fieldName, BytesRef utf8Value) { try (BytesStreamOutput out = new BytesStreamOutput()) { out.write((byte) 's'); /* @@ -290,15 +291,16 @@ public void addString(String fieldName, BytesRef utf8Value) { } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); } + return this; } @Override - public void addIp(String fieldName, InetAddress value) { - addString(fieldName, NetworkAddress.format(value)); + public DocumentDimensions addIp(String fieldName, InetAddress value) { + return addString(fieldName, NetworkAddress.format(value)); } @Override - public void addLong(String fieldName, long value) { + public DocumentDimensions addLong(String fieldName, long value) { try (BytesStreamOutput out = new BytesStreamOutput()) { out.write((byte) 'l'); out.writeLong(value); @@ -306,10 +308,11 @@ public void addLong(String fieldName, long value) { } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); } + return this; } @Override - public void addUnsignedLong(String fieldName, long value) { + public DocumentDimensions addUnsignedLong(String fieldName, long value) { try (BytesStreamOutput out = new BytesStreamOutput()) { Object ul = DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value); if (ul instanceof Long l) { @@ -320,11 +323,27 @@ public void addUnsignedLong(String fieldName, long value) { out.writeLong(value); } add(fieldName, out.bytes()); + return this; } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); } } + @Override + public DocumentDimensions validate(final IndexSettings settings) { + if (settings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ID_HASHING) + && dimensions.size() > settings.getValue(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING)) { + throw new MapperException( + "Too many dimension fields [" + + dimensions.size() + + "], max [" + + settings.getValue(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING) + + "] dimension fields allowed" + ); + } + return this; + } + private void add(String fieldName, BytesReference encoded) throws IOException { final Dimension dimension = new Dimension(new BytesRef(fieldName), encoded); if (dimensions.contains(dimension)) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java index f09c6f8c036c8..d373d683b73ad 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java @@ -173,7 +173,7 @@ private void addField(DocumentParserContext context, ContentPath path, String cu final String keyedFieldName = FlattenedFieldParser.extractKey(bytesKeyedValue).utf8ToString(); if (fieldType.isDimension() && fieldType.dimensions().contains(keyedFieldName)) { final BytesRef keyedFieldValue = FlattenedFieldParser.extractValue(bytesKeyedValue); - context.getDimensions().addString(rootFieldName + "." + keyedFieldName, keyedFieldValue); + context.getDimensions().addString(rootFieldName + "." + keyedFieldName, keyedFieldValue).validate(context.indexSettings()); } } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 97ffd50d5b8c3..ff9838f0cd355 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -633,7 +633,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } if (dimension && numericValue != null) { - context.getDimensions().addUnsignedLong(fieldType().name(), numericValue); + context.getDimensions().addUnsignedLong(fieldType().name(), numericValue).validate(context.indexSettings()); } List fields = new ArrayList<>(); From a612d2242c2a184955dc1db246d73170cc3a2dcf Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 21 Dec 2023 13:05:59 +0100 Subject: [PATCH 100/125] fix: two tests --- .../support/TimeSeriesDimensionsLimitIT.java | 2 +- .../test/downsample/10_basic.yml | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java index 3b8f3f05d9c04..43f29fa80dff5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java @@ -94,7 +94,7 @@ public void testTotalNumberOfDimensionFieldsLimit() { } public void testTotalNumberOfDimensionFieldsDefaultLimit() { - int dimensionFieldLimit = 9999; + int dimensionFieldLimit = 21; final Exception ex = expectThrows(IllegalArgumentException.class, () -> createTimeSeriesIndex(mapping -> { mapping.startObject("routing_field").field("type", "keyword").field("time_series_dimension", true).endObject(); for (int i = 0; i < dimensionFieldLimit; i++) { diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index ea454e59957d3..44d8f0cec3d94 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -322,18 +322,18 @@ setup: - match: { hits.hits.0._source.metricset: pod } - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:00:00.000Z } - - match: { hits.hits.0._source.k8s.pod.multi-counter: 21 } - - match: { hits.hits.0._source.k8s.pod.scaled-counter: 20.0 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 90 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 200 } - - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 726 } + - match: { hits.hits.0._source.k8s.pod.multi-counter: 0 } + - match: { hits.hits.0._source.k8s.pod.scaled-counter: 0.00 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.min: 100 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.max: 102 } + - match: { hits.hits.0._source.k8s.pod.multi-gauge.sum: 607 } - match: { hits.hits.0._source.k8s.pod.multi-gauge.value_count: 6 } - - match: { hits.hits.0._source.k8s.pod.scaled-gauge.min: 90.0 } - - match: { hits.hits.0._source.k8s.pod.scaled-gauge.max: 100.0 } - - match: { hits.hits.0._source.k8s.pod.scaled-gauge.sum: 190.0 } + - match: { hits.hits.0._source.k8s.pod.scaled-gauge.min: 100.0 } + - match: { hits.hits.0._source.k8s.pod.scaled-gauge.max: 101.0 } + - match: { hits.hits.0._source.k8s.pod.scaled-gauge.sum: 201.0 } - match: { hits.hits.0._source.k8s.pod.scaled-gauge.value_count: 2 } - - match: { hits.hits.0._source.k8s.pod.network.tx.min: 2001818691 } - - match: { hits.hits.0._source.k8s.pod.network.tx.max: 2005177954 } + - match: { hits.hits.0._source.k8s.pod.network.tx.min: 1434521831 } + - match: { hits.hits.0._source.k8s.pod.network.tx.max: 1434577921 } - match: { hits.hits.0._source.k8s.pod.network.tx.value_count: 2 } - match: { hits.hits.0._source.k8s.pod.ip: "10.10.55.56" } - match: { hits.hits.0._source.k8s.pod.created_at: "2021-04-28T19:43:00.000Z" } From a7629e8df62fe3f2c6e961090b94c32806866292 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 21 Dec 2023 14:48:37 +0100 Subject: [PATCH 101/125] fix: remove match_all queries --- .../test/tsdb/25_id_generation.yml | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 4069f601fe4ef..1a103c9361764 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -276,15 +276,6 @@ ids query: version: " - 8.12.99" reason: _tsid hasing introduced in 8.13 - - do: - search: - index: test - body: - query: - match_all: {} - - - match: { hits.total.value: 8 } - - do: search: index: test @@ -307,15 +298,6 @@ get: version: " - 8.12.99" reason: _tsid hasing introduced in 8.13 - - do: - search: - index: test - body: - query: - match_all: {} - - - match: { hits.total.value: 8 } - - do: get: index: test @@ -403,15 +385,6 @@ delete over _bulk: version: " - 8.1.99" reason: ids generation changed in 8.2 - - do: - search: - index: test - body: - query: - match_all: {} - - - match: { hits.total.value: 8 } - # mget call added to investigate test failure: https://github.com/elastic/elasticsearch/issues/93852 # (should be removed when test issue is resolved) - do: From 0f3620f5e4450fda474ca2810e367df87882c2cb Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 9 Jan 2024 11:18:23 +0100 Subject: [PATCH 102/125] fix: address review comments --- .../elasticsearch/search/DocValueFormat.java | 2 +- .../mapper/IdLoaderTsidHashingTests.java | 275 -------- .../mapper/TimeSeriesIdFieldMapperTests.java | 656 ------------------ .../downsample/DownsampleShardIndexer.java | 7 - 4 files changed, 1 insertion(+), 939 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java delete mode 100644 server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index ea5de46bbc462..5955059929084 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -699,7 +699,7 @@ public Object format(BytesRef value) { // NOTE: if the tsid is a map of dimension key/value pairs (as it was before introducing // tsid hashing) we just decode the map and return it. return TimeSeriesIdFieldMapper.decodeTsidAsMap(value); - } catch (Throwable t) { + } catch (Exception e) { // NOTE: otherwise the _tsid field is just a hash and we can't decode it return TimeSeriesIdFieldMapper.encodeTsid(value); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java deleted file mode 100644 index 686fb592a7faf..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTsidHashingTests.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.index.mapper; - -import org.apache.lucene.document.LongPoint; -import org.apache.lucene.document.SortedDocValuesField; -import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.document.SortedSetDocValuesField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.SortedNumericSortField; -import org.apache.lucene.store.Directory; -import org.apache.lucene.tests.analysis.MockAnalyzer; -import org.apache.lucene.tests.util.LuceneTestCase; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.routing.IndexRouting; -import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.IntStream; - -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; - -public class IdLoaderTsidHashingTests extends ESTestCase { - - public void testSynthesizeIdSimple() throws Exception { - var routingPaths = List.of("dim1"); - var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); - - long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); - List docs = List.of( - new Doc(startTime, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), - new Doc(startTime + 1, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))), - new Doc(startTime + 2, List.of(new Dimension("dim1", "bbb"), new Dimension("dim2", "xxx"))) - ); - CheckedConsumer verify = indexReader -> { - assertThat(indexReader.leaves(), hasSize(1)); - LeafReader leafReader = indexReader.leaves().get(0).reader(); - assertThat(leafReader.numDocs(), equalTo(3)); - var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 1, 2 }); - assertThat(leaf.getId(0), equalTo(expectedId(routing, docs.get(2)))); - assertThat(leaf.getId(1), equalTo(expectedId(routing, docs.get(0)))); - assertThat(leaf.getId(2), equalTo(expectedId(routing, docs.get(1)))); - }; - prepareIndexReader(indexAndForceMerge(routing, docs), verify, false); - } - - public void testSynthesizeIdMultipleSegments() throws Exception { - var routingPaths = List.of("dim1"); - var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); - - long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); - List docs1 = List.of( - new Doc(startTime, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), - new Doc(startTime - 1, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), - new Doc(startTime - 2, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))), - new Doc(startTime - 3, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "xxx"))) - ); - List docs2 = List.of( - new Doc(startTime, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))), - new Doc(startTime - 1, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))), - new Doc(startTime - 2, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))), - new Doc(startTime - 3, List.of(new Dimension("dim1", "aaa"), new Dimension("dim2", "yyy"))) - ); - List docs3 = List.of( - new Doc(startTime, List.of(new Dimension("dim1", "bbb"), new Dimension("dim2", "yyy"))), - new Doc(startTime - 1, List.of(new Dimension("dim1", "bbb"), new Dimension("dim2", "yyy"))), - new Doc(startTime - 2, List.of(new Dimension("dim1", "bbb"), new Dimension("dim2", "yyy"))), - new Doc(startTime - 3, List.of(new Dimension("dim1", "bbb"), new Dimension("dim2", "yyy"))) - ); - CheckedConsumer buildIndex = writer -> { - for (Doc doc : docs1) { - indexDoc(routing, writer, doc); - } - writer.flush(); - for (Doc doc : docs2) { - indexDoc(routing, writer, doc); - } - writer.flush(); - for (Doc doc : docs3) { - indexDoc(routing, writer, doc); - } - writer.flush(); - }; - CheckedConsumer verify = indexReader -> { - assertThat(indexReader.leaves(), hasSize(3)); - { - LeafReader leafReader = indexReader.leaves().get(0).reader(); - assertThat(leafReader.numDocs(), equalTo(docs1.size())); - var leaf = idLoader.leaf(null, leafReader, IntStream.range(0, docs1.size()).toArray()); - for (int i = 0; i < docs1.size(); i++) { - assertThat(leaf.getId(i), equalTo(expectedId(routing, docs1.get(i)))); - } - } - { - LeafReader leafReader = indexReader.leaves().get(1).reader(); - assertThat(leafReader.numDocs(), equalTo(docs2.size())); - var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 3 }); - assertThat(leaf.getId(0), equalTo(expectedId(routing, docs2.get(0)))); - assertThat(leaf.getId(3), equalTo(expectedId(routing, docs2.get(3)))); - } - { - LeafReader leafReader = indexReader.leaves().get(2).reader(); - assertThat(leafReader.numDocs(), equalTo(docs3.size())); - var leaf = idLoader.leaf(null, leafReader, new int[] { 1, 2 }); - assertThat(leaf.getId(1), equalTo(expectedId(routing, docs3.get(1)))); - assertThat(leaf.getId(2), equalTo(expectedId(routing, docs3.get(2)))); - } - { - LeafReader leafReader = indexReader.leaves().get(2).reader(); - assertThat(leafReader.numDocs(), equalTo(docs3.size())); - var leaf = idLoader.leaf(null, leafReader, new int[] { 3 }); - expectThrows(IllegalArgumentException.class, () -> leaf.getId(0)); - } - }; - prepareIndexReader(buildIndex, verify, true); - } - - public void testSynthesizeIdRandom() throws Exception { - var routingPaths = List.of("dim1"); - var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); - - long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); - Set expectedIDs = new HashSet<>(); - List randomDocs = new ArrayList<>(); - int numberOfTimeSeries = randomIntBetween(8, 64); - for (int i = 0; i < numberOfTimeSeries; i++) { - int numberOfDimensions = randomIntBetween(1, 6); - List dimensions = new ArrayList<>(numberOfDimensions); - for (int j = 1; j <= numberOfDimensions; j++) { - String fieldName = "dim" + j; - Object value; - if (j == 5) { - value = randomLongBetween(1, 20); - } else { - value = randomAlphaOfLength(4); - } - dimensions.add(new Dimension(fieldName, value)); - } - int numberOfSamples = randomIntBetween(1, 16); - for (int j = 0; j < numberOfSamples; j++) { - Doc doc = new Doc(startTime++, dimensions); - randomDocs.add(doc); - expectedIDs.add(expectedId(routing, doc)); - } - } - CheckedConsumer verify = indexReader -> { - assertThat(indexReader.leaves(), hasSize(1)); - LeafReader leafReader = indexReader.leaves().get(0).reader(); - assertThat(leafReader.numDocs(), equalTo(randomDocs.size())); - var leaf = idLoader.leaf(null, leafReader, IntStream.range(0, randomDocs.size()).toArray()); - for (int i = 0; i < randomDocs.size(); i++) { - String actualId = leaf.getId(i); - assertTrue("docId=" + i + " id=" + actualId, expectedIDs.remove(actualId)); - } - }; - prepareIndexReader(indexAndForceMerge(routing, randomDocs), verify, false); - assertThat(expectedIDs, empty()); - } - - private static CheckedConsumer indexAndForceMerge(IndexRouting.ExtractFromSource routing, List docs) { - return writer -> { - for (Doc doc : docs) { - indexDoc(routing, writer, doc); - } - writer.forceMerge(1); - }; - } - - private void prepareIndexReader( - CheckedConsumer buildIndex, - CheckedConsumer verify, - boolean noMergePolicy - ) throws IOException { - try (Directory directory = newDirectory()) { - IndexWriterConfig config = LuceneTestCase.newIndexWriterConfig(random(), new MockAnalyzer(random())); - if (noMergePolicy) { - config.setMergePolicy(NoMergePolicy.INSTANCE); - config.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); - } - Sort sort = new Sort( - new SortField(TimeSeriesIdFieldMapper.NAME, SortField.Type.STRING, false), - new SortedNumericSortField(DataStreamTimestampFieldMapper.DEFAULT_PATH, SortField.Type.LONG, true) - ); - config.setIndexSort(sort); - IndexWriter indexWriter = new IndexWriter(directory, config); - buildIndex.accept(indexWriter); - indexWriter.close(); - - try (DirectoryReader indexReader = DirectoryReader.open(directory);) { - verify.accept(indexReader); - } - } - } - - private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter iw, Doc doc) throws IOException { - final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routing.builder()); - - final List fields = new ArrayList<>(); - fields.add(new SortedNumericDocValuesField(DataStreamTimestampFieldMapper.DEFAULT_PATH, doc.timestamp)); - fields.add(new LongPoint(DataStreamTimestampFieldMapper.DEFAULT_PATH, doc.timestamp)); - for (Dimension dimension : doc.dimensions) { - if (dimension.value instanceof Number n) { - builder.addLong(dimension.field, n.longValue()); - fields.add(new SortedNumericDocValuesField(dimension.field, ((Number) dimension.value).longValue())); - } else { - builder.addString(dimension.field, dimension.value.toString()); - fields.add(new SortedSetDocValuesField(dimension.field, new BytesRef(dimension.value.toString()))); - } - } - BytesRef tsid = builder.buildTsidHash().toBytesRef(); - fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, tsid)); - iw.addDocument(fields); - } - - private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc) throws IOException { - var routingBuilder = routing.builder(); - var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routingBuilder); - for (Dimension dimension : doc.dimensions) { - if (dimension.value instanceof Number n) { - timeSeriesIdBuilder.addLong(dimension.field, n.longValue()); - } else { - timeSeriesIdBuilder.addString(dimension.field, dimension.value.toString()); - } - } - return TsidExtractingIdFieldMapper.createId( - false, - routingBuilder, - timeSeriesIdBuilder.buildTsidHash().toBytesRef(), - doc.timestamp, - new byte[16] - ); - } - - private static IndexRouting.ExtractFromSource createRouting(List routingPaths) { - var settings = indexSettings(IndexVersion.current(), 2, 1).put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2000-01-01T00:00:00.000Z") - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2001-01-01T00:00:00.000Z") - .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPaths) - .build(); - var indexMetadata = IndexMetadata.builder("index").settings(settings).build(); - return (IndexRouting.ExtractFromSource) IndexRouting.fromIndexMetadata(indexMetadata); - } - - record Doc(long timestamp, List dimensions) {} - - record Dimension(String field, Object value) {} - -} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java deleted file mode 100644 index d4dc03d22441b..0000000000000 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ /dev/null @@ -1,656 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.index.mapper; - -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.io.stream.ByteArrayStreamInput; -import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; -import org.elasticsearch.test.index.IndexVersionUtils; -import org.elasticsearch.xcontent.XContentBuilder; - -import java.io.IOException; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.nullValue; - -public class TimeSeriesIdFieldMapperTests extends MetadataMapperTestCase { - - @Override - protected String fieldName() { - return TimeSeriesIdFieldMapper.NAME; - } - - @Override - protected boolean isConfigurable() { - return false; - } - - @Override - protected void registerParameters(ParameterChecker checker) throws IOException { - // There aren't any parameters - } - - @Override - protected IndexVersion getVersion() { - return IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.V_8_8_0, - IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_ID_HASHING) - ); - } - - private DocumentMapper createDocumentMapper(String routingPath, XContentBuilder mappings) throws IOException { - return createMapperService( - getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), 200) // Allow tests that use many dimensions - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath) - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") - .build(), - mappings - ).documentMapper(); - } - - private static ParsedDocument parseDocument(DocumentMapper docMapper, CheckedConsumer f) - throws IOException { - // Add the @timestamp field required by DataStreamTimestampFieldMapper for all time series indices - return docMapper.parse(source(null, b -> { - f.accept(b); - b.field("@timestamp", "2021-10-01"); - }, null)); - } - - private static BytesRef parseAndGetTsid(DocumentMapper docMapper, CheckedConsumer f) throws IOException { - return parseDocument(docMapper, f).rootDoc().getBinaryValue(TimeSeriesIdFieldMapper.NAME); - } - - @SuppressWarnings("unchecked") - public void testEnabledInTimeSeriesMode() throws Exception { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "long").field("time_series_dimension", true).endObject(); - })); - - ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100).field("c", 500)); - assertThat( - doc.rootDoc().getBinaryValue("_tsid"), - equalTo(new BytesRef("\u0002\u0001as\u0005value\u0001bl\u0000\u0000\u0000\u0000\u0000\u0000\u0000d")) - ); - assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); - assertThat(doc.rootDoc().getField("b").numericValue(), equalTo(100L)); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWE"); - } - - public void testDisabledInStandardMode() throws Exception { - DocumentMapper docMapper = createMapperService( - getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()).build(), - mapping(b -> {}) - ).documentMapper(); - assertThat(docMapper.metadataMapper(TimeSeriesIdFieldMapper.class), is(nullValue())); - - ParsedDocument doc = docMapper.parse(source("id", b -> b.field("field", "value"), null)); - assertThat(doc.rootDoc().getBinaryValue("_tsid"), is(nullValue())); - assertThat(doc.rootDoc().get("field"), equalTo("value")); - } - - public void testIncludeInDocumentNotAllowed() throws Exception { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("_tsid", "foo"))); - - assertThat(e.getCause().getMessage(), containsString("Field [_tsid] is a metadata field and cannot be added inside a document")); - } - - /** - * Test with non-randomized string for sanity checking. - */ - @SuppressWarnings("unchecked") - public void testStrings() throws IOException { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("o") - .startObject("properties") - .startObject("e") - .field("type", "keyword") - .field("time_series_dimension", true) - .endObject() - .endObject() - .endObject(); - })); - - BytesRef tsid = parseAndGetTsid( - docMapper, - b -> b.field("a", "foo").field("b", "bar").field("c", "baz").startObject("o").field("e", "bort").endObject() - ); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWE"); - } - - @SuppressWarnings("unchecked") - public void testUnicodeKeys() throws IOException { - String fire = new String(new int[] { 0x1F525 }, 0, 1); - String coffee = "\u2615"; - DocumentMapper docMapper = createDocumentMapper(fire + "," + coffee, mapping(b -> { - b.startObject(fire).field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject(coffee).field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - - ParsedDocument doc = parseDocument(docMapper, b -> b.field(fire, "hot").field(coffee, "good")); - Object tsid = TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); - assertEquals(tsid, "A-I"); - } - - @SuppressWarnings("unchecked") - public void testKeywordTooLong() throws IOException { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - - ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); - } - - @SuppressWarnings("unchecked") - public void testKeywordTooLongUtf8() throws IOException { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - - String theWordLong = "長い"; - ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); - } - - public void testKeywordNull() throws IOException { - DocumentMapper docMapper = createDocumentMapper("r", mapping(b -> { - b.startObject("r").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - - BytesRef withNull = parseAndGetTsid(docMapper, b -> b.field("r", "foo").field("a", (String) null)); - BytesRef withoutField = parseAndGetTsid(docMapper, b -> b.field("r", "foo")); - assertThat(withNull, equalTo(withoutField)); - } - - /** - * Test with non-randomized longs for sanity checking. - */ - @SuppressWarnings("unchecked") - public void testLong() throws IOException { - DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { - b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("a").field("type", "long").field("time_series_dimension", true).endObject(); - b.startObject("o") - .startObject("properties") - .startObject("e") - .field("type", "long") - .field("time_series_dimension", true) - .endObject() - .endObject() - .endObject(); - })); - - BytesRef tsid = parseAndGetTsid(docMapper, b -> { - b.field("kw", "kw"); - b.field("a", 1L); - b.field("b", -1); - b.field("c", "baz"); - b.startObject("o").field("e", 1234).endObject(); - }); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); - } - - public void testLongInvalidString() throws IOException { - DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { - b.startObject("a").field("type", "long").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_a_long"))); - assertThat( - e.getMessage(), - // TODO describe the document instead of "null" - equalTo("[1:6] failed to parse field [a] of type [long] in a time series document. Preview of field's value: 'not_a_long'") - ); - } - - public void testLongNull() throws IOException { - DocumentMapper docMapper = createDocumentMapper("r", mapping(b -> { - b.startObject("r").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("a").field("type", "long").field("time_series_dimension", true).endObject(); - })); - - BytesRef withNull = parseAndGetTsid(docMapper, b -> b.field("r", "foo").field("a", (Long) null)); - BytesRef withoutField = parseAndGetTsid(docMapper, b -> b.field("r", "foo")); - assertThat(withNull, equalTo(withoutField)); - } - - /** - * Test with non-randomized integers for sanity checking. - */ - @SuppressWarnings("unchecked") - public void testInteger() throws IOException { - DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { - b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("a").field("type", "integer").field("time_series_dimension", true).endObject(); - b.startObject("o") - .startObject("properties") - .startObject("e") - .field("type", "integer") - .field("time_series_dimension", true) - .endObject() - .endObject() - .endObject(); - })); - - BytesRef tsid = parseAndGetTsid(docMapper, b -> { - b.field("kw", "kw"); - b.field("a", 1L); - b.field("b", -1); - b.field("c", "baz"); - b.startObject("o").field("e", Integer.MIN_VALUE).endObject(); - }); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); - } - - public void testIntegerInvalidString() throws IOException { - DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { - b.startObject("a").field("type", "integer").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_an_int"))); - assertThat( - e.getMessage(), - equalTo("[1:6] failed to parse field [a] of type [integer] in a time series document. Preview of field's value: 'not_an_int'") - ); - } - - public void testIntegerOutOfRange() throws IOException { - DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { - b.startObject("a").field("type", "integer").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", Long.MAX_VALUE))); - assertThat( - e.getMessage(), - equalTo( - "[1:6] failed to parse field [a] of type [integer] in a time series document. Preview of field's value: '" - + Long.MAX_VALUE - + "'" - ) - ); - } - - /** - * Test with non-randomized shorts for sanity checking. - */ - @SuppressWarnings("unchecked") - public void testShort() throws IOException { - DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { - b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("a").field("type", "short").field("time_series_dimension", true).endObject(); - b.startObject("o") - .startObject("properties") - .startObject("e") - .field("type", "short") - .field("time_series_dimension", true) - .endObject() - .endObject() - .endObject(); - })); - - BytesRef tsid = parseAndGetTsid(docMapper, b -> { - b.field("kw", "kw"); - b.field("a", 1L); - b.field("b", -1); - b.field("c", "baz"); - b.startObject("o").field("e", Short.MIN_VALUE).endObject(); - }); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); - } - - public void testShortInvalidString() throws IOException { - DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { - b.startObject("a").field("type", "short").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_a_short"))); - assertThat( - e.getMessage(), - equalTo("[1:6] failed to parse field [a] of type [short] in a time series document. Preview of field's value: 'not_a_short'") - ); - } - - public void testShortOutOfRange() throws IOException { - DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { - b.startObject("a").field("type", "short").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", Long.MAX_VALUE))); - assertThat( - e.getMessage(), - equalTo( - "[1:6] failed to parse field [a] of type [short] in a time series document. Preview of field's value: '" - + Long.MAX_VALUE - + "'" - ) - ); - } - - /** - * Test with non-randomized shorts for sanity checking. - */ - @SuppressWarnings("unchecked") - public void testByte() throws IOException { - DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { - b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("a").field("type", "byte").field("time_series_dimension", true).endObject(); - b.startObject("o") - .startObject("properties") - .startObject("e") - .field("type", "byte") - .field("time_series_dimension", true) - .endObject() - .endObject() - .endObject(); - })); - - BytesRef tsid = parseAndGetTsid(docMapper, b -> { - b.field("kw", "kw"); - b.field("a", 1L); - b.field("b", -1); - b.field("c", "baz"); - b.startObject("o").field("e", (int) Byte.MIN_VALUE).endObject(); - }); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); - } - - public void testByteInvalidString() throws IOException { - DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { - b.startObject("a").field("type", "byte").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_a_byte"))); - assertThat( - e.getMessage(), - equalTo("[1:6] failed to parse field [a] of type [byte] in a time series document. Preview of field's value: 'not_a_byte'") - ); - } - - public void testByteOutOfRange() throws IOException { - DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { - b.startObject("a").field("type", "byte").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", Long.MAX_VALUE))); - assertThat( - e.getMessage(), - equalTo( - "[1:6] failed to parse field [a] of type [byte] in a time series document. Preview of field's value: '" - + Long.MAX_VALUE - + "'" - ) - ); - } - - /** - * Test with non-randomized ips for sanity checking. - */ - @SuppressWarnings("unchecked") - public void testIp() throws IOException { - DocumentMapper docMapper = createDocumentMapper("kw", mapping(b -> { - b.startObject("kw").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("a").field("type", "ip").field("time_series_dimension", true).endObject(); - b.startObject("o") - .startObject("properties") - .startObject("e") - .field("type", "ip") - .field("time_series_dimension", true) - .endObject() - .endObject() - .endObject(); - })); - - ParsedDocument doc = parseDocument(docMapper, b -> { - b.field("kw", "kw"); - b.field("a", "192.168.0.1"); - b.field("b", -1); - b.field("c", "baz"); - b.startObject("o").field("e", "255.255.255.1").endObject(); - }); - assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWFz"); - } - - public void testIpInvalidString() throws IOException { - DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { - b.startObject("a").field("type", "ip").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); - })); - Exception e = expectThrows(DocumentParsingException.class, () -> parseDocument(docMapper, b -> b.field("a", "not_an_ip"))); - assertThat( - e.getMessage(), - equalTo("[1:6] failed to parse field [a] of type [ip] in a time series document. Preview of field's value: 'not_an_ip'") - ); - } - - /** - * Tests when the total of the tsid is more than 32k. - */ - @SuppressWarnings("unchecked") - public void testVeryLarge() throws IOException { - DocumentMapper docMapper = createDocumentMapper("b", mapping(b -> { - b.startObject("b").field("type", "keyword").field("time_series_dimension", true).endObject(); - for (int i = 0; i < 100; i++) { - b.startObject("d" + i).field("type", "keyword").field("time_series_dimension", true).endObject(); - } - })); - - String large = "many words ".repeat(80); - ParsedDocument doc = parseDocument(docMapper, b -> { - b.field("b", "foo"); - for (int i = 0; i < 100; i++) { - b.field("d" + i, large); - } - }); - - Object tsid = TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); - assertEquals( - tsid, - "AWJzA2ZvbwJkMHPwBm1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd" - + "29yZHMgbWFueSB3b3JkcyA" - ); - } - - /** - * Sending the same document twice produces the same value. - */ - public void testSameGenConsistentForSameDoc() throws IOException { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); - b.startObject("c").field("type", "long").field("time_series_dimension", true).endObject(); - })); - - String a = randomAlphaOfLength(10); - int b = between(1, 100); - int c = between(0, 2); - CheckedConsumer fields = d -> d.field("a", a).field("b", b).field("c", (long) c); - ParsedDocument doc1 = parseDocument(docMapper, fields); - ParsedDocument doc2 = parseDocument(docMapper, fields); - assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); - } - - /** - * Non-dimension fields don't influence the value of _tsid. - */ - public void testExtraFieldsDoNotMatter() throws IOException { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); - b.startObject("c").field("type", "long").field("time_series_dimension", true).endObject(); - })); - - String a = randomAlphaOfLength(10); - int b = between(1, 100); - int c = between(0, 2); - ParsedDocument doc1 = parseDocument( - docMapper, - d -> d.field("a", a).field("b", b).field("c", (long) c).field("e", between(10, 100)) - ); - ParsedDocument doc2 = parseDocument( - docMapper, - d -> d.field("a", a).field("b", b).field("c", (long) c).field("e", between(50, 200)) - ); - assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); - } - - /** - * The order that the dimensions appear in the document do not influence the value of _tsid. - */ - public void testOrderDoesNotMatter() throws IOException { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); - b.startObject("c").field("type", "long").field("time_series_dimension", true).endObject(); - })); - - String a = randomAlphaOfLength(10); - int b = between(1, 100); - int c = between(0, 2); - ParsedDocument doc1 = parseDocument(docMapper, d -> d.field("a", a).field("b", b).field("c", (long) c)); - ParsedDocument doc2 = parseDocument(docMapper, d -> d.field("b", b).field("a", a).field("c", (long) c)); - assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); - } - - /** - * Dimensions that appear in the mapping but not in the document don't influence the value of _tsid. - */ - public void testUnusedExtraDimensions() throws IOException { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); - b.startObject("c").field("type", "long").field("time_series_dimension", true).endObject(); - })); - - String a = randomAlphaOfLength(10); - int b = between(1, 100); - CheckedConsumer fields = d -> d.field("a", a).field("b", b); - ParsedDocument doc1 = parseDocument(docMapper, fields); - ParsedDocument doc2 = parseDocument(docMapper, fields); - assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); - } - - /** - * Different values for dimensions change the result. - */ - public void testDifferentValues() throws IOException { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); - })); - - String a = randomAlphaOfLength(10); - ParsedDocument doc1 = parseDocument(docMapper, d -> d.field("a", a).field("b", between(1, 100))); - ParsedDocument doc2 = parseDocument(docMapper, d -> d.field("a", a + 1).field("b", between(200, 300))); - assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); - } - - public void testSameMetricNamesDifferentValues() throws IOException { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); - b.startObject("m1").field("type", "double").field("time_series_metric", "gauge").endObject(); - b.startObject("m2").field("type", "integer").field("time_series_metric", "counter").endObject(); - })); - - ParsedDocument doc1 = parseDocument( - docMapper, - d -> d.field("a", "value") - .field("b", 10) - .field("m1", randomDoubleBetween(100, 200, true)) - .field("m2", randomIntBetween(100, 200)) - ); - ParsedDocument doc2 = parseDocument( - docMapper, - d -> d.field("a", "value").field("b", 10).field("m1", randomDoubleBetween(10, 20, true)).field("m2", randomIntBetween(10, 20)) - ); - assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); - } - - public void testDifferentMetricNamesSameValues() throws IOException { - DocumentMapper docMapper1 = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); - b.startObject("m1").field("type", "double").field("time_series_metric", "gauge").endObject(); - })); - - DocumentMapper docMapper2 = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); - b.startObject("m2").field("type", "double").field("time_series_metric", "gauge").endObject(); - })); - - double metricValue = randomDoubleBetween(10, 20, true); - ParsedDocument doc1 = parseDocument(docMapper1, d -> d.field("a", "value").field("b", 10).field("m1", metricValue)); - ParsedDocument doc2 = parseDocument(docMapper2, d -> d.field("a", "value").field("b", 10).field("m2", metricValue)); - // NOTE: plain tsid (not hashed) does not take metric names/values into account - assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); - } - - /** - * Two documents with the same *values* but different dimension keys will generate - * different {@code _tsid}s. - */ - public void testDifferentDimensions() throws IOException { - // First doc mapper has dimension fields a and b - DocumentMapper docMapper1 = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); - })); - // Second doc mapper has dimension fields a and c - DocumentMapper docMapper2 = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("c").field("type", "integer").field("time_series_dimension", true).endObject(); - })); - - String a = randomAlphaOfLength(10); - int b = between(1, 100); - int c = between(5, 500); - CheckedConsumer fields = d -> d.field("a", a).field("b", b).field("c", c); - ParsedDocument doc1 = parseDocument(docMapper1, fields); - ParsedDocument doc2 = parseDocument(docMapper2, fields); - assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); - } - - /** - * Documents with fewer dimensions have a different value. - */ - public void testFewerDimensions() throws IOException { - DocumentMapper docMapper = createDocumentMapper("a", mapping(b -> { - b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); - b.startObject("b").field("type", "integer").field("time_series_dimension", true).endObject(); - b.startObject("c").field("type", "integer").field("time_series_dimension", true).endObject(); - })); - - String a = randomAlphaOfLength(10); - int b = between(1, 100); - int c = between(5, 500); - ParsedDocument doc1 = parseDocument(docMapper, d -> d.field("a", a).field("b", b)); - ParsedDocument doc2 = parseDocument(docMapper, d -> d.field("a", a).field("b", b).field("c", c)); - assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, not(doc2.rootDoc().getBinaryValue("_tsid").bytes)); - } -} diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java index 6137ba23d9fe3..844c644ee9ea6 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/DownsampleShardIndexer.java @@ -551,13 +551,6 @@ public XContentBuilder buildDownsampleDocument() throws IOException { } builder.field(timestampField.name(), timestampFormat.format(timestamp)); builder.field(DocCountFieldMapper.NAME, docCount); - // TODO: should we make sure extracting dimension fields is backward compatible with older indices versions? - // Extract dimension values from _tsid field, so we avoid loading them from doc_values - // Map dimensions = (Map) DocValueFormat.TIME_SERIES_ID.format(tsid); - // for (Map.Entry e : dimensions.entrySet()) { - // assert e.getValue() != null; - // builder.field((String) e.getKey(), e.getValue()); - // } // Serialize fields for (DownsampleFieldSerializer fieldProducer : groupedProducers) { From df9cb54af3b97edfc68023bbdb39d6c1ec99a598 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 9 Jan 2024 11:55:15 +0100 Subject: [PATCH 103/125] fix: remove unused import --- .../xpack/downsample/DownsampleActionSingleNodeTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index 6c36b5fd7171b..4088f9fefc1ed 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -59,7 +59,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregations; From 2d748bba7dc1cda647b54863d2fe2d4864358290 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 9 Jan 2024 11:59:12 +0100 Subject: [PATCH 104/125] restore original file --- ...java => TimeSeriesIdFieldMapperTests.java} | 76 +++++++------------ 1 file changed, 28 insertions(+), 48 deletions(-) rename server/src/test/java/org/elasticsearch/index/mapper/{TimeSeriesIdFieldMapperTsidHashingTests.java => TimeSeriesIdFieldMapperTests.java} (90%) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java similarity index 90% rename from server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index dd6a5504892e2..d4dc03d22441b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTsidHashingTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -28,7 +28,7 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; -public class TimeSeriesIdFieldMapperTsidHashingTests extends MetadataMapperTestCase { +public class TimeSeriesIdFieldMapperTests extends MetadataMapperTestCase { @Override protected String fieldName() { @@ -47,7 +47,11 @@ protected void registerParameters(ParameterChecker checker) throws IOException { @Override protected IndexVersion getVersion() { - return IndexVersionUtils.randomVersionBetween(random(), IndexVersions.TIME_SERIES_ID_HASHING, IndexVersion.current()); + return IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.V_8_8_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_ID_HASHING) + ); } private DocumentMapper createDocumentMapper(String routingPath, XContentBuilder mappings) throws IOException { @@ -83,10 +87,13 @@ public void testEnabledInTimeSeriesMode() throws Exception { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "value").field("b", 100).field("c", 500)); - assertEquals( - "30CO74tuRyatuMXEvNJvbu4OOPE_BPY6XRQDNqrFfRGnHvkCRkdF1g", - TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) + assertThat( + doc.rootDoc().getBinaryValue("_tsid"), + equalTo(new BytesRef("\u0002\u0001as\u0005value\u0001bl\u0000\u0000\u0000\u0000\u0000\u0000\u0000d")) ); + assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); + assertThat(doc.rootDoc().getField("b").numericValue(), equalTo(100L)); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWE"); } public void testDisabledInStandardMode() throws Exception { @@ -131,10 +138,7 @@ public void testStrings() throws IOException { docMapper, b -> b.field("a", "foo").field("b", "bar").field("c", "baz").startObject("o").field("e", "bort").endObject() ); - assertEquals( - "S3oqtElkKwgW-mWA1nr9gK_2Gi60D5a0AuSISsBYPmjw-qWQ7YI-3A", - TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWE"); } @SuppressWarnings("unchecked") @@ -147,10 +151,8 @@ public void testUnicodeKeys() throws IOException { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field(fire, "hot").field(coffee, "good")); - assertEquals( - "A-nLhW-M69syDAiaaHLtCp6MBeMMqVDqgBVXpLViGUOdgCAZniSp6A", - TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) - ); + Object tsid = TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); + assertEquals(tsid, "A-I"); } @SuppressWarnings("unchecked") @@ -160,10 +162,7 @@ public void testKeywordTooLong() throws IOException { })); ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", "more_than_1024_bytes".repeat(52))); - assertEquals( - "FlVJQe85E3Mv4Ekz_gxSa71kNSgI7Z7KeQP6VlsQZzmG4ohj", - TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); } @SuppressWarnings("unchecked") @@ -174,10 +173,7 @@ public void testKeywordTooLongUtf8() throws IOException { String theWordLong = "長い"; ParsedDocument doc = parseDocument(docMapper, b -> b.field("a", theWordLong.repeat(200))); - assertEquals( - "FlVJQe85E3Mv4Ekz_gxSa0zjd_qfoPhsUMih0si8XDmP_VV7", - TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AQ"); } public void testKeywordNull() throws IOException { @@ -216,10 +212,7 @@ public void testLong() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", 1234).endObject(); }); - assertEquals( - "iI8WAECVhaC5BYgDlkz8OK2bjW9KuknL3XBrYDfJ_rolBiDD3_cY-Gu6cUA", - TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testLongInvalidString() throws IOException { @@ -271,10 +264,7 @@ public void testInteger() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", Integer.MIN_VALUE).endObject(); }); - assertEquals( - "iI8WAECVhaC5BYgDlkz8OK2bjW9KuknL1fIRUB0-Ns-fIV82SmmaBGvMBFc", - TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testIntegerInvalidString() throws IOException { @@ -330,10 +320,7 @@ public void testShort() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", Short.MIN_VALUE).endObject(); }); - assertEquals( - "iI8WAECVhaC5BYgDlkz8OK2bjW9KuknLJgBZj8kLWyz4_fSeI5hcDbH2Quo", - TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testShortInvalidString() throws IOException { @@ -389,10 +376,7 @@ public void testByte() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", (int) Byte.MIN_VALUE).endObject(); }); - assertEquals( - "iI8WAECVhaC5BYgDlkz8OK2bjW9KuknLKCuqhxFhxsmpAlmUjJPOtj1L020", - TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new BytesArray(tsid).streamInput()), "AWFs"); } public void testByteInvalidString() throws IOException { @@ -448,10 +432,7 @@ public void testIp() throws IOException { b.field("c", "baz"); b.startObject("o").field("e", "255.255.255.1").endObject(); }); - assertEquals( - "iI8WAECVhaC5BYgDlkz8OAus6VZKuknLs_NO7CBqqZuzDo7PTQiv_wLD1eY", - TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) - ); + assertEquals(TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)), "AWFz"); } public void testIpInvalidString() throws IOException { @@ -485,14 +466,12 @@ public void testVeryLarge() throws IOException { b.field("d" + i, large); } }); + + Object tsid = TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)); assertEquals( - "QZ1K9tk9UIcpA826eU6H-a_2Gi5YqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKl" - + "Bk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZN" - + "YqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKl" - + "Bk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZN" - + "YqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKl" - + "Bk1ipQZNYqUGTWKlBk1ipQZNYqUGTWKlBk1ipQZNYqUGTxUG_ass830ZjEXx7y5eiMg", - TimeSeriesIdFieldMapper.encodeTsid(new ByteArrayStreamInput(doc.rootDoc().getBinaryValue("_tsid").bytes)) + tsid, + "AWJzA2ZvbwJkMHPwBm1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd29yZHMgbWFueSB3b3JkcyBtYW55IHdvcmRzIG1hbnkgd" + + "29yZHMgbWFueSB3b3JkcyA" ); } @@ -628,6 +607,7 @@ public void testDifferentMetricNamesSameValues() throws IOException { double metricValue = randomDoubleBetween(10, 20, true); ParsedDocument doc1 = parseDocument(docMapper1, d -> d.field("a", "value").field("b", 10).field("m1", metricValue)); ParsedDocument doc2 = parseDocument(docMapper2, d -> d.field("a", "value").field("b", 10).field("m2", metricValue)); + // NOTE: plain tsid (not hashed) does not take metric names/values into account assertThat(doc1.rootDoc().getBinaryValue("_tsid").bytes, equalTo(doc2.rootDoc().getBinaryValue("_tsid").bytes)); } From becde73841db190251aace2d4f91fbd807352266 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 10 Jan 2024 12:37:57 +0100 Subject: [PATCH 105/125] fix: remove unused encoder --- .../src/main/java/org/elasticsearch/search/DocValueFormat.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 5955059929084..51b2e62159a4d 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -670,8 +670,6 @@ public double parseDouble(String value, boolean roundUp, LongSupplier now) { * DocValues format for time series id. */ class TimeSeriesIdDocValueFormat implements DocValueFormat { - - private static final Base64.Encoder BASE64_ENCODER = Base64.getUrlEncoder().withoutPadding(); private static final Base64.Decoder BASE64_DECODER = Base64.getUrlDecoder(); private TimeSeriesIdDocValueFormat() {} From f523408f2a66c9325e0dc0e7d30b58dedf574a48 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 10 Jan 2024 12:38:19 +0100 Subject: [PATCH 106/125] fix: catch Throwable to catch AssertionError in non-release builds --- .../main/java/org/elasticsearch/search/DocValueFormat.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 51b2e62159a4d..c3370dd4d8ec5 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -697,8 +697,11 @@ public Object format(BytesRef value) { // NOTE: if the tsid is a map of dimension key/value pairs (as it was before introducing // tsid hashing) we just decode the map and return it. return TimeSeriesIdFieldMapper.decodeTsidAsMap(value); - } catch (Exception e) { + } catch (Throwable t) { // NOTE: otherwise the _tsid field is just a hash and we can't decode it + // We need to catch a Throwable to be able to catch an AssertionError which is thrown + // if decoding of the tsid fails in UnicodeUtil#UTF8toUTF16 which happens if we try to decode + // a tsid hash as a map. return TimeSeriesIdFieldMapper.encodeTsid(value); } } From 7d505423b51b6e72d525467ff2c944f4cf52b8f3 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 10 Jan 2024 12:46:41 +0100 Subject: [PATCH 107/125] fix: catch the assertion error when calling utf8ToString --- .../index/mapper/TimeSeriesIdFieldMapper.java | 10 ++++++++-- .../java/org/elasticsearch/search/DocValueFormat.java | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index b4c2d61fb2630..b8f1d4643ec75 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -381,8 +381,14 @@ public static Map decodeTsidAsMap(StreamInput in) { int type = in.read(); switch (type) { - case (byte) 's' -> // parse a string - result.put(name, in.readBytesRef().utf8ToString()); + case (byte) 's' -> { + // parse a string + try { + result.put(name, in.readBytesRef().utf8ToString()); + } catch (AssertionError ae) { + throw new IllegalArgumentException("Error parsing keyword dimension " + name + ": " + ae.getMessage(), ae); + } + } case (byte) 'l' -> // parse a long result.put(name, in.readLong()); case (byte) 'u' -> { // parse an unsigned_long diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index c3370dd4d8ec5..30be41304b3a0 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -697,7 +697,7 @@ public Object format(BytesRef value) { // NOTE: if the tsid is a map of dimension key/value pairs (as it was before introducing // tsid hashing) we just decode the map and return it. return TimeSeriesIdFieldMapper.decodeTsidAsMap(value); - } catch (Throwable t) { + } catch (Exception e) { // NOTE: otherwise the _tsid field is just a hash and we can't decode it // We need to catch a Throwable to be able to catch an AssertionError which is thrown // if decoding of the tsid fails in UnicodeUtil#UTF8toUTF16 which happens if we try to decode From 5f1d08bc36611e28231e1dc24cd9c222df0f2ed2 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 10 Jan 2024 14:50:23 +0100 Subject: [PATCH 108/125] fix: catch AssertionError while parsing keyword fields --- .../index/mapper/TimeSeriesIdFieldMapper.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index b8f1d4643ec75..d7b8a9a49970e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -377,7 +377,12 @@ public static Map decodeTsidAsMap(StreamInput in) { Map result = new LinkedHashMap<>(size); for (int i = 0; i < size; i++) { - String name = in.readBytesRef().utf8ToString(); + String name = null; + try { + name = in.readBytesRef().utf8ToString(); + } catch (AssertionError ae) { + throw new IllegalArgumentException("Error parsing keyword dimension: " + ae.getMessage(), ae); + } int type = in.read(); switch (type) { @@ -386,7 +391,7 @@ public static Map decodeTsidAsMap(StreamInput in) { try { result.put(name, in.readBytesRef().utf8ToString()); } catch (AssertionError ae) { - throw new IllegalArgumentException("Error parsing keyword dimension " + name + ": " + ae.getMessage(), ae); + throw new IllegalArgumentException("Error parsing keyword dimension: " + ae.getMessage(), ae); } } case (byte) 'l' -> // parse a long From faf76d6f2b072162c60978bfbae280a7c179b484 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 10 Jan 2024 15:28:27 +0100 Subject: [PATCH 109/125] comment: remove comment no longer relevant --- .../src/main/java/org/elasticsearch/search/DocValueFormat.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 30be41304b3a0..51b2e62159a4d 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -699,9 +699,6 @@ public Object format(BytesRef value) { return TimeSeriesIdFieldMapper.decodeTsidAsMap(value); } catch (Exception e) { // NOTE: otherwise the _tsid field is just a hash and we can't decode it - // We need to catch a Throwable to be able to catch an AssertionError which is thrown - // if decoding of the tsid fails in UnicodeUtil#UTF8toUTF16 which happens if we try to decode - // a tsid hash as a map. return TimeSeriesIdFieldMapper.encodeTsid(value); } } From cd01b8c70dda2f964e6050d7877d85783339bbbe Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 15 Jan 2024 18:20:09 +0100 Subject: [PATCH 110/125] test: include test on number of buckets --- .../TimeSeriesTsidHashCardinalityIT.java | 373 ++++++++++++++++++ .../DownsampleActionSingleNodeTests.java | 2 +- 2 files changed, 374 insertions(+), 1 deletion(-) create mode 100644 modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java new file mode 100644 index 0000000000000..1200e5c67ff4f --- /dev/null +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java @@ -0,0 +1,373 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.aggregations.bucket; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.aggregations.AggregationsPlugin; +import org.elasticsearch.aggregations.bucket.timeseries.InternalTimeSeries; +import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.InternalCardinality; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; + +public class TimeSeriesTsidHashCardinalityIT extends ESSingleNodeTestCase { + private static final String START_TIME = "2021-01-01T00:00:00Z"; + private static final String END_TIME = "2021-12-31T23:59:59Z"; + private String beforeIndex, afterIndex; + private long startTime, endTime; + private int docCount, dimCardinality; + private int numDimensions, numTimeSeries; + + @Override + protected Collection> getPlugins() { + List> plugins = new ArrayList<>(super.getPlugins()); + plugins.add(InternalSettingsPlugin.class); + plugins.add(AggregationsPlugin.class); + return plugins; + } + + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + @Override + public void setUp() throws Exception { + super.setUp(); + beforeIndex = randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + afterIndex = randomAlphaOfLength(12).toLowerCase(Locale.ROOT); + startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(START_TIME); + endTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(END_TIME); + docCount = 50_000; + numTimeSeries = 12_000; + dimCardinality = 5; + // NOTE: we need to use few dimensions to be able to index documents in an index created before introducing TSID hashing + numDimensions = randomIntBetween(10, 20); + + final Settings.Builder settings = indexSettings(1, 0).put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) + .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), List.of("dim_routing")) + .put( + IndexSettings.TIME_SERIES_START_TIME.getKey(), + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(Instant.ofEpochMilli(startTime).toEpochMilli()) + ) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), END_TIME); + + final XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties"); + mapping.startObject("@timestamp").field("type", "date").endObject(); + + // Dimensions + mapping.startObject("dim_routing").field("type", "keyword").field("time_series_dimension", true).endObject(); + for (int i = 1; i <= numDimensions; i++) { + mapping.startObject("dim_" + i).field("type", "keyword").field("time_series_dimension", true).endObject(); + } + // Metrics + mapping.startObject("gauge").field("type", "double").field("time_series_metric", "gauge").endObject(); + mapping.endObject().endObject().endObject(); + assertAcked( + indicesAdmin().prepareCreate(beforeIndex) + .setSettings( + settings.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersions.NEW_INDEXVERSION_FORMAT).build() + ) + .setMapping(mapping) + .get() + ); + + assertAcked( + indicesAdmin().prepareCreate(afterIndex) + .setSettings( + settings.put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), IndexVersions.TIME_SERIES_ID_HASHING).build() + ) + .setMapping(mapping) + .get() + ); + + final TimeSeriesDataset timeSeriesDataset = new TimeSeriesDataset(); + while (timeSeriesDataset.size() < numTimeSeries) { + final Set dimensions = new TreeSet<>(); + for (int j = 0; j < numDimensions; j++) { + if (randomIntBetween(1, 10) >= 2) { // NOTE: 20% chance the dimension field is empty + dimensions.add(new Dimension("dim_" + j, randomAlphaOfLength(12))); + } + } + final TimeSeries ts = new TimeSeries(dimensions); + if (timeSeriesDataset.exists(ts)) { + continue; + } + for (int k = 0; k < randomIntBetween(100, 200); k++) { + ts.addValue(randomLongBetween(startTime, endTime), randomDoubleBetween(100.0D, 200.0D, true)); + } + timeSeriesDataset.add(ts); + } + + final BulkRequestBuilder beforeBulkIndexRequest = client().prepareBulk(); + final BulkRequestBuilder afterBulkIndexRequest = client().prepareBulk(); + for (final TimeSeries ts : timeSeriesDataset) { + for (final TimeSeriesValue timeSeriesValue : ts.values) { + final XContentBuilder docSource = XContentFactory.jsonBuilder(); + docSource.startObject(); + docSource.field("dim_routing", "foo"); // Just to make sure we have at least one routing dimension + for (int d = 0; d < numDimensions; d++) { + final Dimension dimension = ts.getDimension("dim_" + d); + if (dimension != null) { + docSource.field("dim_" + d, dimension.value); + } + } + docSource.field("@timestamp", timeSeriesValue.timestamp); + docSource.field("gauge", timeSeriesValue.gauge); + docSource.endObject(); + beforeBulkIndexRequest.add(prepareIndex(beforeIndex).setOpType(DocWriteRequest.OpType.CREATE).setSource(docSource)); + afterBulkIndexRequest.add(prepareIndex(afterIndex).setOpType(DocWriteRequest.OpType.CREATE).setSource(docSource)); + } + } + assertFalse(beforeBulkIndexRequest.get().hasFailures()); + assertFalse(afterBulkIndexRequest.get().hasFailures()); + assertEquals(RestStatus.OK, indicesAdmin().prepareRefresh(beforeIndex, afterIndex).get().getStatus()); + } + + public void testTimeSeriesIdCardinality() { + final SearchResponse searchBefore = client().prepareSearch(beforeIndex) + .addAggregation(new CardinalityAggregationBuilder("card").field("_tsid").precisionThreshold(40000)) + .get(); + final SearchResponse searchAfter = client().prepareSearch(afterIndex) + .addAggregation(new CardinalityAggregationBuilder("card").field("_tsid").precisionThreshold(40000)) + .get(); + + assertEquals( + Arrays.stream(admin().indices().prepareStats(beforeIndex).get().getShards()).findFirst().get().getStats().docs.getCount(), + Arrays.stream(admin().indices().prepareStats(afterIndex).get().getShards()).findFirst().get().getStats().docs.getCount() + ); + assertEquals( + Arrays.stream(admin().indices().prepareStats(beforeIndex).get().getShards()).findFirst().get().getStats().docs.getCount(), + docCount + ); + assertEquals( + Arrays.stream(admin().indices().prepareStats(afterIndex).get().getShards()).findFirst().get().getStats().docs.getCount(), + docCount + ); + + final InternalCardinality cardinalityBefore = searchBefore.getAggregations().get("card"); + final InternalCardinality cardinalityAfter = searchAfter.getAggregations().get("card"); + assertEquals(cardinalityBefore.getValue(), cardinalityAfter.getValue()); + } + + public void testTimeSeriesNumberOfBuckets() { + final SearchResponse searchBefore = client().prepareSearch(beforeIndex) + .setSize(0) + .addAggregation(new TimeSeriesAggregationBuilder("ts")) + .get(); + final SearchResponse searchAfter = client().prepareSearch(afterIndex) + .setSize(0) + .addAggregation(new TimeSeriesAggregationBuilder("ts")) + .get(); + final InternalTimeSeries beforeTimeSeries = searchBefore.getAggregations().get("ts"); + final InternalTimeSeries afterTimeSeries = searchAfter.getAggregations().get("ts"); + assertEquals(beforeTimeSeries.getBuckets().size(), afterTimeSeries.getBuckets().size()); + } + + static class Dimension implements Comparable { + private final String name, value; + + Dimension(final String name, final String value) { + this.name = name; + this.value = value; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Dimension dimension = (Dimension) o; + return Objects.equals(name, dimension.name) && Objects.equals(value, dimension.value); + } + + @Override + public int hashCode() { + return Objects.hash(name, value); + } + + @Override + public String toString() { + return "Dimension{" + "name='" + name + '\'' + ", value='" + value + '\'' + '}'; + } + + @Override + public int compareTo(final Dimension that) { + return Comparator.comparing(Dimension::getName).thenComparing(Dimension::getValue).compare(this, that); + } + } + + static class TimeSeriesValue { + private final long timestamp; + private final double gauge; + + TimeSeriesValue(long timestamp, double gauge) { + this.timestamp = timestamp; + this.gauge = gauge; + } + + public long getTimestamp() { + return timestamp; + } + + public double getGauge() { + return gauge; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TimeSeriesValue that = (TimeSeriesValue) o; + return timestamp == that.timestamp && Double.compare(gauge, that.gauge) == 0; + } + + @Override + public int hashCode() { + return Objects.hash(timestamp, gauge); + } + + @Override + public String toString() { + return "TimeSeriesValue{" + "timestamp=" + timestamp + ", gauge=" + gauge + '}'; + } + } + + static class TimeSeries { + private final HashMap dimensions; + private final Set values; + + TimeSeries(final Set dimensions) { + this.dimensions = new HashMap<>(); + for (final Dimension dimension : dimensions) { + this.dimensions.put(dimension.name, dimension); + } + values = new TreeSet<>(Comparator.comparing(TimeSeriesValue::getTimestamp)); + } + + public void addValue(long timestamp, double gauge) { + values.add(new TimeSeriesValue(timestamp, gauge)); + } + + public String id() { + final StringBuilder sb = new StringBuilder(); + for (final Dimension dimension : dimensions.values().stream().sorted(Comparator.comparing(Dimension::getName)).toList()) { + sb.append(dimension.getName()).append("=").append(dimension.getValue()); + } + + return sb.toString(); + } + + public Dimension getDimension(final String name) { + return this.dimensions.get(name); + } + } + + private class TimeSeriesDataset implements Iterable { + private final HashMap dataset; + + TimeSeriesDataset() { + this.dataset = new HashMap<>(); + } + + public void add(final TimeSeries ts) { + TimeSeries previous = dataset.put(ts.id(), ts); + if (previous != null) { + throw new IllegalArgumentException("Existing time series: " + ts.id()); + } + } + + public boolean exists(final TimeSeries ts) { + return dataset.containsKey(ts.id()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TimeSeriesDataset that = (TimeSeriesDataset) o; + return Objects.equals(dataset, that.dataset); + } + + @Override + public int hashCode() { + return Objects.hash(dataset); + } + + @Override + public String toString() { + return "TimeSeriesDataset{" + "dataset=" + dataset + '}'; + } + + @Override + public Iterator iterator() { + return new TimeSeriesIterator(this.dataset.entrySet()); + } + + public int size() { + return this.dataset.size(); + } + + class TimeSeriesIterator implements Iterator { + + private final Iterator> it; + + TimeSeriesIterator(final Set> entries) { + this.it = entries.iterator(); + } + + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public TimeSeries next() { + return it.next().getValue(); + } + } + } +} diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index 4088f9fefc1ed..b0320c642fc1f 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -243,7 +243,7 @@ public void setup() throws IOException { public void testDownsampleIndex() throws IOException { DownsampleConfig config = new DownsampleConfig(randomInterval()); - SourceSupplier sourceSupplier = () -> { + SourceSupplier sourcatceSupplier = () -> { String ts = randomDateForInterval(config.getInterval()); double labelDoubleValue = DATE_FORMATTER.parseMillis(ts); int labelIntegerValue = randomInt(); From d719bed48440339584df572c4b4146db81741061 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 16 Jan 2024 09:42:40 +0100 Subject: [PATCH 111/125] fix: remove cardinality test and change some settings --- .../TimeSeriesTsidHashCardinalityIT.java | 39 +++---------------- 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java index 1200e5c67ff4f..2623df4807ef7 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java @@ -51,7 +51,7 @@ public class TimeSeriesTsidHashCardinalityIT extends ESSingleNodeTestCase { private static final String END_TIME = "2021-12-31T23:59:59Z"; private String beforeIndex, afterIndex; private long startTime, endTime; - private int docCount, dimCardinality; + private int docCount; private int numDimensions, numTimeSeries; @Override @@ -74,9 +74,8 @@ public void setUp() throws Exception { afterIndex = randomAlphaOfLength(12).toLowerCase(Locale.ROOT); startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(START_TIME); endTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(END_TIME); - docCount = 50_000; - numTimeSeries = 12_000; - dimCardinality = 5; + docCount = 130_000; + numTimeSeries = 15_000; // NOTE: we need to use few dimensions to be able to index documents in an index created before introducing TSID hashing numDimensions = randomIntBetween(10, 20); @@ -129,7 +128,7 @@ public void setUp() throws Exception { if (timeSeriesDataset.exists(ts)) { continue; } - for (int k = 0; k < randomIntBetween(100, 200); k++) { + for (int k = 0; k < randomIntBetween(10, 20); k++) { ts.addValue(randomLongBetween(startTime, endTime), randomDoubleBetween(100.0D, 200.0D, true)); } timeSeriesDataset.add(ts); @@ -160,32 +159,6 @@ public void setUp() throws Exception { assertEquals(RestStatus.OK, indicesAdmin().prepareRefresh(beforeIndex, afterIndex).get().getStatus()); } - public void testTimeSeriesIdCardinality() { - final SearchResponse searchBefore = client().prepareSearch(beforeIndex) - .addAggregation(new CardinalityAggregationBuilder("card").field("_tsid").precisionThreshold(40000)) - .get(); - final SearchResponse searchAfter = client().prepareSearch(afterIndex) - .addAggregation(new CardinalityAggregationBuilder("card").field("_tsid").precisionThreshold(40000)) - .get(); - - assertEquals( - Arrays.stream(admin().indices().prepareStats(beforeIndex).get().getShards()).findFirst().get().getStats().docs.getCount(), - Arrays.stream(admin().indices().prepareStats(afterIndex).get().getShards()).findFirst().get().getStats().docs.getCount() - ); - assertEquals( - Arrays.stream(admin().indices().prepareStats(beforeIndex).get().getShards()).findFirst().get().getStats().docs.getCount(), - docCount - ); - assertEquals( - Arrays.stream(admin().indices().prepareStats(afterIndex).get().getShards()).findFirst().get().getStats().docs.getCount(), - docCount - ); - - final InternalCardinality cardinalityBefore = searchBefore.getAggregations().get("card"); - final InternalCardinality cardinalityAfter = searchAfter.getAggregations().get("card"); - assertEquals(cardinalityBefore.getValue(), cardinalityAfter.getValue()); - } - public void testTimeSeriesNumberOfBuckets() { final SearchResponse searchBefore = client().prepareSearch(beforeIndex) .setSize(0) @@ -306,7 +279,7 @@ public Dimension getDimension(final String name) { } } - private class TimeSeriesDataset implements Iterable { + static class TimeSeriesDataset implements Iterable { private final HashMap dataset; TimeSeriesDataset() { @@ -351,7 +324,7 @@ public int size() { return this.dataset.size(); } - class TimeSeriesIterator implements Iterator { + static class TimeSeriesIterator implements Iterator { private final Iterator> it; From 1831af393fe8d73d281c1169485144ce545ecb0b Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 18 Jan 2024 17:27:36 +0100 Subject: [PATCH 112/125] bring back InternalTimeSeries assertions --- .../aggregations/bucket/timeseries/InternalTimeSeries.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java index f56c294a5bd0d..67a7773fd01bb 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeries.java @@ -205,6 +205,7 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurr ? ((TimeSeriesAggregationBuilder) reduceContext.builder()).getSize() : null; // tests may use a fake builder List bucketsWithSameKey = new ArrayList<>(aggregations.size()); + BytesRef prevTsid = null; while (pq.size() > 0) { reduceContext.consumeBucketsAndMaybeBreak(1); bucketsWithSameKey.clear(); @@ -230,10 +231,13 @@ protected boolean lessThan(IteratorAndCurrent a, IteratorAndCurr } else { reducedBucket = reduceBucket(bucketsWithSameKey, reduceContext); } + BytesRef tsid = reducedBucket.key; + assert prevTsid == null || tsid.compareTo(prevTsid) > 0; reduced.buckets.add(reducedBucket); if (size != null && reduced.buckets.size() >= size) { break; } + prevTsid = tsid; } return reduced; } From 196f6bf3e7b83f6d5883717fff12779f08d49904 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 23 Jan 2024 11:02:22 +0100 Subject: [PATCH 113/125] fix: checkstyle warnings --- .../aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java index 2623df4807ef7..ec3169b60f324 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java @@ -22,8 +22,6 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.InternalCardinality; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.xcontent.XContentBuilder; @@ -31,7 +29,6 @@ import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Comparator; import java.util.HashMap; From d0969598ea7eade176a3c73e08d229a9578cf2a4 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 23 Jan 2024 11:25:47 +0100 Subject: [PATCH 114/125] fix: incorrect variable name --- .../xpack/downsample/DownsampleActionSingleNodeTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index b0320c642fc1f..4088f9fefc1ed 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -243,7 +243,7 @@ public void setup() throws IOException { public void testDownsampleIndex() throws IOException { DownsampleConfig config = new DownsampleConfig(randomInterval()); - SourceSupplier sourcatceSupplier = () -> { + SourceSupplier sourceSupplier = () -> { String ts = randomDateForInterval(config.getInterval()); double labelDoubleValue = DATE_FORMATTER.parseMillis(ts); int labelIntegerValue = randomInt(); From 54727e43b45572e40bb7fa22aa87ebeecd59c17a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 23 Jan 2024 11:55:09 +0100 Subject: [PATCH 115/125] fix: indexing pressure too high --- .../aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java index ec3169b60f324..483443b004e35 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java @@ -48,7 +48,6 @@ public class TimeSeriesTsidHashCardinalityIT extends ESSingleNodeTestCase { private static final String END_TIME = "2021-12-31T23:59:59Z"; private String beforeIndex, afterIndex; private long startTime, endTime; - private int docCount; private int numDimensions, numTimeSeries; @Override @@ -71,8 +70,7 @@ public void setUp() throws Exception { afterIndex = randomAlphaOfLength(12).toLowerCase(Locale.ROOT); startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(START_TIME); endTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis(END_TIME); - docCount = 130_000; - numTimeSeries = 15_000; + numTimeSeries = 5_000; // NOTE: we need to use few dimensions to be able to index documents in an index created before introducing TSID hashing numDimensions = randomIntBetween(10, 20); From a58f97bc0f465a01e67262ad8ac303d9d78ebdd6 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 23 Jan 2024 14:50:20 +0100 Subject: [PATCH 116/125] fix: release search response --- .../downsample/DownsampleActionSingleNodeTests.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index 4088f9fefc1ed..05578becf1d0b 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -59,6 +59,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchResponseUtils; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregations; @@ -975,12 +976,9 @@ public void testResumeDownsamplePartial() throws IOException { ); final DownsampleIndexerAction.ShardDownsampleResponse response2 = indexer.execute(); - int dim2DocCount = client().prepareSearch(sourceIndex) - .setQuery(new TermQueryBuilder(FIELD_DIMENSION_1, "dim1")) - .setSize(10_000) - .get() - .getHits() - .getHits().length; + long dim2DocCount = SearchResponseUtils.getTotalHitsValue( + client().prepareSearch(sourceIndex).setQuery(new TermQueryBuilder(FIELD_DIMENSION_1, "dim1")).setSize(10_000) + ); assertDownsampleIndexer(indexService, shardNum, task, response2, dim2DocCount); } From 608ce2d53f2489c3ec44fef11f4c1c50047451ba Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 23 Jan 2024 16:18:56 +0100 Subject: [PATCH 117/125] fix: release responses --- ...riesAggregationsUnlimitedDimensionsIT.java | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java index c6d9a692e92c0..da92430f7e757 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -149,8 +149,11 @@ private static XContentBuilder timeSeriesIndexMapping() throws IOException { public void testTimeSeriesAggregation() { final TimeSeriesAggregationBuilder timeSeries = new TimeSeriesAggregationBuilder("ts"); final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); - final InternalTimeSeries ts = (InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0); - assertTimeSeriesAggregation(ts); + try { + assertTimeSeriesAggregation((InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0)); + } finally { + aggregationResponse.decRef(); + } } public void testSumByTsid() { @@ -160,8 +163,12 @@ public void testSumByTsid() { final SearchResponse searchResponse = client().prepareSearch("index").setQuery(new MatchAllQueryBuilder()).get(); assertNotEquals(numberOfDocuments, searchResponse.getHits().getHits().length); final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); - final InternalTimeSeries ts = (InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0); - assertTimeSeriesAggregation(ts); + try { + assertTimeSeriesAggregation((InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0)); + } finally { + searchResponse.decRef(); + aggregationResponse.decRef(); + } } public void testTermsByTsid() { @@ -169,8 +176,11 @@ public void testTermsByTsid() { new TermsAggregationBuilder("terms").field("dim_0") ); final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); - final InternalTimeSeries ts = (InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0); - assertTimeSeriesAggregation(ts); + try { + assertTimeSeriesAggregation((InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0)); + } finally { + aggregationResponse.decRef(); + } } public void testDateHistogramByTsid() { @@ -178,8 +188,11 @@ public void testDateHistogramByTsid() { new DateHistogramAggregationBuilder("date_histogram").field("@timestamp").calendarInterval(DateHistogramInterval.MINUTE) ); final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); - final InternalTimeSeries ts = (InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0); - assertTimeSeriesAggregation(ts); + try { + assertTimeSeriesAggregation((InternalTimeSeries) aggregationResponse.getAggregations().asList().get(0)); + } finally { + aggregationResponse.decRef(); + } } public void testCardinalityByTsid() { @@ -187,9 +200,13 @@ public void testCardinalityByTsid() { new CardinalityAggregationBuilder("dim_n_cardinality").field("dim_" + (numberOfDimensions - 1)) ); final SearchResponse aggregationResponse = client().prepareSearch("index").addAggregation(timeSeries).setSize(0).get(); - ((InternalTimeSeries) aggregationResponse.getAggregations().get("ts")).getBuckets().forEach(bucket -> { - assertCardinality(bucket.getAggregations().get("dim_n_cardinality"), 1); - }); + try { + ((InternalTimeSeries) aggregationResponse.getAggregations().get("ts")).getBuckets().forEach(bucket -> { + assertCardinality(bucket.getAggregations().get("dim_n_cardinality"), 1); + }); + } finally { + aggregationResponse.decRef(); + } } private static void assertTimeSeriesAggregation(final InternalTimeSeries timeSeriesAggregation) { From 4d9a4a44cc25aedd66743cac4932d223dfecba43 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 23 Jan 2024 16:26:44 +0100 Subject: [PATCH 118/125] fix: release responses --- .../bucket/TimeSeriesTsidHashCardinalityIT.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java index 483443b004e35..eea28bd002448 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java @@ -163,9 +163,14 @@ public void testTimeSeriesNumberOfBuckets() { .setSize(0) .addAggregation(new TimeSeriesAggregationBuilder("ts")) .get(); - final InternalTimeSeries beforeTimeSeries = searchBefore.getAggregations().get("ts"); - final InternalTimeSeries afterTimeSeries = searchAfter.getAggregations().get("ts"); - assertEquals(beforeTimeSeries.getBuckets().size(), afterTimeSeries.getBuckets().size()); + try { + final InternalTimeSeries beforeTimeSeries = searchBefore.getAggregations().get("ts"); + final InternalTimeSeries afterTimeSeries = searchAfter.getAggregations().get("ts"); + assertEquals(beforeTimeSeries.getBuckets().size(), afterTimeSeries.getBuckets().size()); + } finally { + searchBefore.decRef(); + searchAfter.decRef(); + } } static class Dimension implements Comparable { From acf167846ddc8d636ca19be6db92de756b7376ef Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 29 Jan 2024 15:04:37 +0100 Subject: [PATCH 119/125] fix: increase dimension limit default setting --- .../bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java | 1 - .../aggregations/bucket/TimeSeriesNestedAggregationsIT.java | 1 - .../main/java/org/elasticsearch/index/mapper/MapperService.java | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java index da92430f7e757..18b24123e6cf0 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesAggregationsUnlimitedDimensionsIT.java @@ -109,7 +109,6 @@ private CreateIndexResponse prepareTimeSeriesIndex( .put("index.number_of_replicas", randomIntBetween(1, 3)) .put("time_series.start_time", startMillis) .put("time_series.end_time", endMillis) - .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), numberOfDimensions + 1) .put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), 4192) .build() ).setMapping(mapping).get(); diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java index a2ff9bd85f38b..3287f50ab1739 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesNestedAggregationsIT.java @@ -110,7 +110,6 @@ private CreateIndexResponse prepareTimeSeriesIndex( .put("index.number_of_replicas", randomIntBetween(1, 3)) .put("time_series.start_time", startMillis) .put("time_series.end_time", endMillis) - .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), numberOfDimensions + 1) .put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), 4192) .build() ).setMapping(mapping).get(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index b714eabbd2636..b532a87b99e75 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -116,7 +116,7 @@ public enum MergeReason { ); public static final Setting INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING = Setting.longSetting( "index.mapping.dimension_fields.limit", - 21, + 32_768, 0, Property.Dynamic, Property.IndexScope From 31d42031da640489b96f51b5098abbc5498e07be Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 31 Jan 2024 15:28:27 +0100 Subject: [PATCH 120/125] fix: total number of fields limit --- .../timeseries/support/TimeSeriesDimensionsLimitIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java index 43f29fa80dff5..2ef951b28d3aa 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/timeseries/support/TimeSeriesDimensionsLimitIT.java @@ -94,7 +94,7 @@ public void testTotalNumberOfDimensionFieldsLimit() { } public void testTotalNumberOfDimensionFieldsDefaultLimit() { - int dimensionFieldLimit = 21; + int dimensionFieldLimit = 32 * 1024; final Exception ex = expectThrows(IllegalArgumentException.class, () -> createTimeSeriesIndex(mapping -> { mapping.startObject("routing_field").field("type", "keyword").field("time_series_dimension", true).endObject(); for (int i = 0; i < dimensionFieldLimit; i++) { @@ -175,7 +175,7 @@ private void createTimeSeriesIndex( Settings.Builder settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) - .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 11_000) + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 38 * 1024) .putList(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPaths.get()) .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2000-01-08T23:40:53.384Z") .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2106-01-08T23:40:53.384Z"); From ef4f9a22fb96740c86e99118dceed3e3a8a05ab8 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 1 Feb 2024 11:31:18 +0100 Subject: [PATCH 121/125] fix: use a record instead of a class --- .../TimeSeriesTsidHashCardinalityIT.java | 72 ++++--------------- 1 file changed, 15 insertions(+), 57 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java index eea28bd002448..f5d8056d71cb8 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java @@ -173,62 +173,20 @@ public void testTimeSeriesNumberOfBuckets() { } } - static class Dimension implements Comparable { - private final String name, value; - - Dimension(final String name, final String value) { - this.name = name; - this.value = value; - } - - public String getName() { - return name; - } - - public String getValue() { - return value; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Dimension dimension = (Dimension) o; - return Objects.equals(name, dimension.name) && Objects.equals(value, dimension.value); - } - - @Override - public int hashCode() { - return Objects.hash(name, value); - } + record Dimension(String name, String value) implements Comparable { @Override - public String toString() { - return "Dimension{" + "name='" + name + '\'' + ", value='" + value + '\'' + '}'; - } - - @Override - public int compareTo(final Dimension that) { - return Comparator.comparing(Dimension::getName).thenComparing(Dimension::getValue).compare(this, that); - } - } - - static class TimeSeriesValue { - private final long timestamp; - private final double gauge; - - TimeSeriesValue(long timestamp, double gauge) { - this.timestamp = timestamp; - this.gauge = gauge; - } + public String toString() { + return "Dimension{" + "name='" + name + '\'' + ", value='" + value + '\'' + '}'; + } - public long getTimestamp() { - return timestamp; + @Override + public int compareTo(final Dimension that) { + return Comparator.comparing(Dimension::name).thenComparing(Dimension::value).compare(this, that); + } } - public double getGauge() { - return gauge; - } + record TimeSeriesValue(long timestamp, double gauge) { @Override public boolean equals(Object o) { @@ -244,10 +202,10 @@ public int hashCode() { } @Override - public String toString() { - return "TimeSeriesValue{" + "timestamp=" + timestamp + ", gauge=" + gauge + '}'; + public String toString() { + return "TimeSeriesValue{" + "timestamp=" + timestamp + ", gauge=" + gauge + '}'; + } } - } static class TimeSeries { private final HashMap dimensions; @@ -258,7 +216,7 @@ static class TimeSeries { for (final Dimension dimension : dimensions) { this.dimensions.put(dimension.name, dimension); } - values = new TreeSet<>(Comparator.comparing(TimeSeriesValue::getTimestamp)); + values = new TreeSet<>(Comparator.comparing(TimeSeriesValue::timestamp)); } public void addValue(long timestamp, double gauge) { @@ -267,8 +225,8 @@ public void addValue(long timestamp, double gauge) { public String id() { final StringBuilder sb = new StringBuilder(); - for (final Dimension dimension : dimensions.values().stream().sorted(Comparator.comparing(Dimension::getName)).toList()) { - sb.append(dimension.getName()).append("=").append(dimension.getValue()); + for (final Dimension dimension : dimensions.values().stream().sorted(Comparator.comparing(Dimension::name)).toList()) { + sb.append(dimension.name()).append("=").append(dimension.value()); } return sb.toString(); From e3a49ba795400e57785ebc6276f497af52e29c9c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 1 Feb 2024 11:31:54 +0100 Subject: [PATCH 122/125] fix: spelling hasing vs hashing --- .../rest-api-spec/test/data_stream/150_tsdb.yml | 4 ++-- .../rest-api-spec/test/delete/70_tsdb.yml | 2 +- .../rest-api-spec/test/tsdb/100_composite.yml | 4 ++-- .../rest-api-spec/test/tsdb/140_routing_path.yml | 4 ++-- .../rest-api-spec/test/tsdb/25_id_generation.yml | 16 ++++++++-------- .../rest-api-spec/test/tsdb/30_snapshot.yml | 2 +- .../rest-api-spec/test/tsdb/40_search.yml | 8 ++++---- .../rest-api-spec/test/tsdb/50_alias.yml | 4 ++-- .../test/tsdb/60_add_dimensions.yml | 10 +++++----- .../test/tsdb/70_dimension_types.yml | 14 +++++++------- .../rest-api-spec/test/tsdb/80_index_resize.yml | 6 +++--- .../rest-api-spec/test/downsample/10_basic.yml | 16 ++++++++-------- .../test/downsample/40_runtime_fields.yml | 6 +++--- .../test/downsample/60_settings.yml | 2 +- .../rest-api-spec/test/downsample/10_basic.yml | 2 +- .../rest-api-spec/test/analytics/100_tsdb.yml | 2 +- 16 files changed, 51 insertions(+), 51 deletions(-) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 42dc83b6e5bef..7663c43e7cc49 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -134,7 +134,7 @@ created the data stream: fetch the tsid: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: @@ -153,7 +153,7 @@ fetch the tsid: "aggregate the tsid": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hahing introduced in 8.13 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml index 207a23b8f212b..83f8aab34e02a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml @@ -2,7 +2,7 @@ "basic tsdb delete": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml index 0f963c11632d3..e546c60c5916a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml @@ -65,7 +65,7 @@ setup: composite aggregation on tsid: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: @@ -115,7 +115,7 @@ composite aggregation on tsid: composite aggregation on tsid with after: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 6f0f846807279..5813445326ef6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -3,7 +3,7 @@ missing routing path field: - skip: features: close_to version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -123,7 +123,7 @@ multi-value routing path field: - skip: features: close_to version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 1a103c9361764..531879cb67d30 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -2,7 +2,7 @@ setup: - skip: version: " - 8.1.99,8.7.00 - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -137,7 +137,7 @@ generates a consistent id: index a new document on top of an old one: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: @@ -259,7 +259,7 @@ create operation on top of old document fails: create operation on top of old document fails over bulk: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: bulk: @@ -274,7 +274,7 @@ create operation on top of old document fails over bulk: ids query: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: @@ -296,7 +296,7 @@ ids query: get: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: get: @@ -346,7 +346,7 @@ get with routing: delete: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: delete: @@ -415,7 +415,7 @@ delete over _bulk: routing_path matches deep object: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -460,7 +460,7 @@ routing_path matches deep object: routing_path matches object: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 1e6dc803e001e..ff925ce56a70c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -2,7 +2,7 @@ setup: - skip: version: "8.7.00 - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: snapshot.create_repository: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index 35fcb22118fbc..5b779894a2cb1 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -2,7 +2,7 @@ setup: - skip: version: " - 8.1.99,8.7.00 - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -172,7 +172,7 @@ fetch a tag: "fetch the tsid": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: @@ -267,7 +267,7 @@ aggregate a tag: "aggregate the tsid": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: @@ -310,7 +310,7 @@ aggregate a tag: sort by tsid: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index 45ba4201ae2e0..829e26a90805a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -65,7 +65,7 @@ setup: search an alias: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.put_alias: @@ -94,7 +94,7 @@ search an alias: index into alias: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.put_alias: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 2354e14dd51cc..b30c03c4797da 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -2,7 +2,7 @@ add dimensions with put_mapping: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -55,7 +55,7 @@ add dimensions with put_mapping: add dimensions to no dims with dynamic_template over index: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -103,7 +103,7 @@ add dimensions to no dims with dynamic_template over index: add dimensions to no dims with dynamic_template over bulk: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -151,7 +151,7 @@ add dimensions to no dims with dynamic_template over bulk: add dimensions to some dims with dynamic_template over index: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -203,7 +203,7 @@ add dimensions to some dims with dynamic_template over index: add dimensions to some dims with dynamic_template over bulk: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 2975fd7b40f5b..6398fee265fef 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -3,7 +3,7 @@ keyword dimension: - skip: features: close_to version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: @@ -77,7 +77,7 @@ flattened dimension: - skip: features: close_to version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: @@ -165,7 +165,7 @@ flattened empty dimension: - skip: features: close_to version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: @@ -246,7 +246,7 @@ flattened field missing routing path field: - skip: features: close_to version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -341,7 +341,7 @@ flattened field misspelled routing path field: - skip: features: close_to version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -433,7 +433,7 @@ long dimension: - skip: features: close_to version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -508,7 +508,7 @@ ip dimension: - skip: features: close_to version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index db4536fdd94be..486e5bf8dc607 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -2,7 +2,7 @@ setup: - skip: version: " - 8.1.99,8.7.00 - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 features: "arbitrary_key" # Force allocating all shards to a single node so that we can shrink later. @@ -103,7 +103,7 @@ split: shrink: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.shrink: @@ -130,7 +130,7 @@ shrink: clone: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.clone: diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index 23901987dca3f..b8443bb2554aa 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -298,7 +298,7 @@ setup: "Downsample index": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.downsample: @@ -732,7 +732,7 @@ setup: "Downsample a downsampled index": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.downsample: @@ -871,7 +871,7 @@ setup: "Downsample histogram as label": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.downsample: @@ -1283,7 +1283,7 @@ setup: "Downsample object field": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.downsample: @@ -1351,7 +1351,7 @@ setup: "Downsample empty and missing labels": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.downsample: @@ -1412,7 +1412,7 @@ setup: "Downsample label with ignore_above": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -1539,7 +1539,7 @@ setup: "Downsample index with empty dimension": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -1622,7 +1622,7 @@ setup: "Downsample index with empty dimension on routing path": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml index 3f27ec6f28a5e..b9dbf621c60dc 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/40_runtime_fields.yml @@ -2,7 +2,7 @@ "Runtime fields accessing metric fields in downsample target index": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 features: close_to - do: @@ -164,7 +164,7 @@ "Runtime field accessing dimension fields in downsample target index": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: @@ -285,7 +285,7 @@ "Runtime field accessing label fields in downsample target index": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.create: diff --git a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml index e272681f6d29c..e533db97b3f7e 100644 --- a/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml +++ b/x-pack/plugin/downsample/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/downsample/60_settings.yml @@ -94,7 +94,7 @@ "Downsample datastream with tier preference": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 features: allowed_warnings - do: diff --git a/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml b/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml index 0084030255761..a1f7ac650141a 100644 --- a/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml +++ b/x-pack/plugin/downsample/qa/with-security/src/yamlRestTest/resources/rest-api-spec/test/downsample/10_basic.yml @@ -294,7 +294,7 @@ setup: "Downsample index": - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: indices.downsample: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml index cf618e183457a..5b2416cdc31a4 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/100_tsdb.yml @@ -64,7 +64,7 @@ setup: aggretate multi_terms: - skip: version: " - 8.12.99" - reason: _tsid hasing introduced in 8.13 + reason: _tsid hashing introduced in 8.13 - do: search: From b0ab308886efd4b453011744cfa156560714b39c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 1 Feb 2024 11:39:20 +0100 Subject: [PATCH 123/125] fix: update Lucene version --- server/src/main/java/org/elasticsearch/index/IndexVersions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 7ad73f1130c7a..ae6185cdcc6b6 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -100,7 +100,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion UPGRADE_8_12_1_LUCENE_9_9_2 = def(8_500_010, Version.LUCENE_9_9_2); public static final IndexVersion NEW_INDEXVERSION_FORMAT = def(8_501_00_0, Version.LUCENE_9_9_1); public static final IndexVersion UPGRADE_LUCENE_9_9_2 = def(8_502_00_0, Version.LUCENE_9_9_2); - public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_502_00_1, Version.LUCENE_9_9_1); + public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_502_00_1, Version.LUCENE_9_9_2); /* * STOP! READ THIS FIRST! No, really, From 76a38c97fe056679f50757c6b7cf6d3a17e6d538 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 1 Feb 2024 11:47:35 +0100 Subject: [PATCH 124/125] fix: checkstyle warnings --- .../TimeSeriesTsidHashCardinalityIT.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java index f5d8056d71cb8..b1db2f8a7d3a1 100644 --- a/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java +++ b/modules/aggregations/src/internalClusterTest/java/org/elasticsearch/aggregations/bucket/TimeSeriesTsidHashCardinalityIT.java @@ -176,15 +176,15 @@ public void testTimeSeriesNumberOfBuckets() { record Dimension(String name, String value) implements Comparable { @Override - public String toString() { - return "Dimension{" + "name='" + name + '\'' + ", value='" + value + '\'' + '}'; - } + public String toString() { + return "Dimension{" + "name='" + name + '\'' + ", value='" + value + '\'' + '}'; + } - @Override - public int compareTo(final Dimension that) { - return Comparator.comparing(Dimension::name).thenComparing(Dimension::value).compare(this, that); - } + @Override + public int compareTo(final Dimension that) { + return Comparator.comparing(Dimension::name).thenComparing(Dimension::value).compare(this, that); } + } record TimeSeriesValue(long timestamp, double gauge) { @@ -202,10 +202,10 @@ public int hashCode() { } @Override - public String toString() { - return "TimeSeriesValue{" + "timestamp=" + timestamp + ", gauge=" + gauge + '}'; - } + public String toString() { + return "TimeSeriesValue{" + "timestamp=" + timestamp + ", gauge=" + gauge + '}'; } + } static class TimeSeries { private final HashMap dimensions; From c53fac50ba16077cc9a970eb3e4f863e822c39b5 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 1 Feb 2024 13:30:39 +0100 Subject: [PATCH 125/125] fix: remove conflict marker --- .../resources/rest-api-spec/test/aggregations/time_series.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 2102914a31915..9e8ec6b3f6768 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -510,4 +510,3 @@ setup: max: max: field: val ->>>>>>> main