diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index c909441db6d54..6de7f0a3aae8b 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -257,6 +257,9 @@ public final class IndexScopedSettings extends AbstractScopedSettings { if (IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG) { settings.add(IndexSettings.DISABLE_SEQUENCE_NUMBERS); } + if (IndexSettings.ALLOW_LARGE_BINARY_BLOCK_SIZE.isEnabled()) { + settings.add(IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BINARY_BLOCK_SIZE); + } settings.add(IndexSettings.INDEX_MAPPING_EXCLUDE_SOURCE_VECTORS_SETTING); BUILT_IN_INDEX_SETTINGS = Collections.unmodifiableSet(settings); }; diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index cebbab5a0de78..639c30299f0fa 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -888,6 +888,14 @@ public Iterator> settings() { Property.Final ); + public static final FeatureFlag ALLOW_LARGE_BINARY_BLOCK_SIZE = new FeatureFlag("allow_large_binary_block_size"); + public static final Setting USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BINARY_BLOCK_SIZE = Setting.boolSetting( + "index.use_time_series_doc_values_format_large_binary_block_size", + false, + Property.IndexScope, + Property.Final + ); + /** * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use. * TODO: Remove in 9.0 @@ -1148,7 +1156,8 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private final boolean useDocValuesSkipperForHostname; private final boolean useTimeSeriesSyntheticId; private final boolean useTimeSeriesDocValuesFormat; - private final boolean useTimeSeriesDocValuesFormatLargeBlockSize; + private final boolean useTimeSeriesDocValuesFormatLargeNumericBlockSize; + private final boolean useTimeSeriesDocValuesFormatLargeBinaryBlockSize; private final boolean useEs812PostingsFormat; private final boolean disableSequenceNumbers; @@ -1355,7 +1364,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti : version.onOrAfter(IndexVersions.SKIPPERS_ENABLED_BY_DEFAULT) && version.before(IndexVersions.SKIPPER_DEFAULTS_ONLY_ON_TSDB); seqNoIndexOptions = scopedSettings.get(SEQ_NO_INDEX_OPTIONS_SETTING); useTimeSeriesDocValuesFormat = scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING); - useTimeSeriesDocValuesFormatLargeBlockSize = scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BLOCK_SIZE); + useTimeSeriesDocValuesFormatLargeNumericBlockSize = scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BLOCK_SIZE); + useTimeSeriesDocValuesFormatLargeBinaryBlockSize = ALLOW_LARGE_BINARY_BLOCK_SIZE.isEnabled() + && scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BINARY_BLOCK_SIZE); useEs812PostingsFormat = scopedSettings.get(USE_ES_812_POSTINGS_FORMAT); intraMergeParallelismEnabled = scopedSettings.get(INTRA_MERGE_PARALLELISM_ENABLED_SETTING); useTimeSeriesSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && scopedSettings.get(SYNTHETIC_ID); @@ -2136,8 +2147,18 @@ public boolean useTimeSeriesDocValuesFormat() { /** * @return Whether the time series doc value format with large numeric block size should be used. */ - public boolean isUseTimeSeriesDocValuesFormatLargeBlockSize() { - return useTimeSeriesDocValuesFormatLargeBlockSize; + public boolean isUseTimeSeriesDocValuesFormatLargeNumericBlockSize() { + return useTimeSeriesDocValuesFormatLargeNumericBlockSize; + } + + /** + * Checks if the time series DocValues format is configured to use a large binary block size. + * + * @return {@code true} if the time series DocValues format is using a large binary block size; + * {@code false} otherwise. + */ + public boolean isUseTimeSeriesDocValuesFormatLargeBinaryBlockSize() { + return useTimeSeriesDocValuesFormatLargeBinaryBlockSize; } /** diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java index 642956959425e..9d842672aeb5b 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java @@ -196,8 +196,9 @@ public DocValuesFormat getDocValuesFormatForField(String field) { if (useTSDBDocValuesFormat(field)) { var indexCreatedVersion = mapperService.getIndexSettings().getIndexVersionCreated(); - boolean useLargeBlockSize = mapperService.getIndexSettings().isUseTimeSeriesDocValuesFormatLargeBlockSize(); - return TSDBDocValuesFormatFactory.createDocValuesFormat(indexCreatedVersion, useLargeBlockSize); + boolean useLargeNumericBlockSize = mapperService.getIndexSettings().isUseTimeSeriesDocValuesFormatLargeNumericBlockSize(); + boolean useLargeBinaryBlockSize = mapperService.getIndexSettings().isUseTimeSeriesDocValuesFormatLargeBinaryBlockSize(); + return TSDBDocValuesFormatFactory.createDocValuesFormat(indexCreatedVersion, useLargeNumericBlockSize, useLargeBinaryBlockSize); } return docValuesFormat; diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumer.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumer.java index 720963774af56..e2198a73720a2 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumer.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumer.java @@ -55,8 +55,6 @@ import java.util.List; import static org.elasticsearch.index.codec.tsdb.es819.DocValuesConsumerUtil.compatibleWithOptimizedMerge; -import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.BLOCK_BYTES_THRESHOLD; -import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.BLOCK_COUNT_THRESHOLD; import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT; import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.SKIP_INDEX_LEVEL_SHIFT; import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.SKIP_INDEX_MAX_LEVEL; @@ -79,6 +77,8 @@ final class ES819TSDBDocValuesConsumer extends XDocValuesConsumer { final BinaryDVCompressionMode binaryDVCompressionMode; private final boolean enablePerBlockCompression; // only false for testing private final DocOffsetsCodec.Encoder docOffsetsEncoder; + private final int blockBytesThreshold; + private final int blockCountThreshold; ES819TSDBDocValuesConsumer( BinaryDVCompressionMode binaryDVCompressionMode, @@ -89,6 +89,8 @@ final class ES819TSDBDocValuesConsumer extends XDocValuesConsumer { int minDocsPerOrdinalForOrdinalRangeEncoding, boolean enableOptimizedMerge, int numericBlockShift, + int blockBytesThreshold, + int blockCountThreshold, String dataCodec, String dataExtension, String metaCodec, @@ -97,6 +99,8 @@ final class ES819TSDBDocValuesConsumer extends XDocValuesConsumer { this.binaryDVCompressionMode = binaryDVCompressionMode; this.enablePerBlockCompression = enablePerBlockCompression; this.docOffsetsEncoder = docOffsetsEncoder; + this.blockBytesThreshold = blockBytesThreshold; + this.blockCountThreshold = blockCountThreshold; this.state = state; this.termsDictBuffer = new byte[1 << 14]; this.dir = state.directory; @@ -534,7 +538,7 @@ public void close() throws IOException { private final class CompressedBinaryBlockWriter implements BinaryWriter { final Compressor compressor; - final int[] docOffsets = new int[BLOCK_COUNT_THRESHOLD + 1]; // start for each doc plus start of doc that would be after last + final int[] docOffsets = new int[blockCountThreshold + 1]; int uncompressedBlockLength = 0; int maxUncompressedBlockLength = 0; @@ -561,7 +565,7 @@ public void addDoc(BytesRef v) throws IOException { numDocsInCurrentBlock++; docOffsets[numDocsInCurrentBlock] = uncompressedBlockLength; - if (uncompressedBlockLength >= BLOCK_BYTES_THRESHOLD || numDocsInCurrentBlock >= BLOCK_COUNT_THRESHOLD) { + if (uncompressedBlockLength >= blockBytesThreshold || numDocsInCurrentBlock >= blockCountThreshold) { flushData(); } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java index df6e8c30b529b..aa7788955a735 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java @@ -69,8 +69,8 @@ public class ES819TSDBDocValuesFormat extends org.apache.lucene.codecs.DocValues * is 128k, or if the number of values is 1024. These values are a tradeoff between the high compression ratio and decompression * speed of large blocks, and the ability to avoid decompressing unneeded values provided by small blocks. */ - public static final int BLOCK_BYTES_THRESHOLD = 128 * 1024; - public static final int BLOCK_COUNT_THRESHOLD = 1024; + public static final int BINARY_DV_BLOCK_BYTES_THRESHOLD_DEFAULT = 128 * 1024; + public static final int BINARY_DV_BLOCK_COUNT_THRESHOLD_DEFAULT = 1024; // number of documents in an interval static final int DEFAULT_SKIP_INDEX_INTERVAL_SIZE = 4096; @@ -140,6 +140,8 @@ private static boolean getOptimizedMergeEnabledDefault() { final BinaryDVCompressionMode binaryDVCompressionMode; final boolean enablePerBlockCompression; final DocOffsetsCodec docOffsetsCodec; + final int blockBytesThreshold; + final int blockCountThreshold; public static ES819TSDBDocValuesFormat getInstance(boolean useLargeNumericBlock) { return useLargeNumericBlock ? new ES819TSDBDocValuesFormat(NUMERIC_LARGE_BLOCK_SHIFT) : new ES819TSDBDocValuesFormat(); @@ -229,6 +231,32 @@ public ES819TSDBDocValuesFormat( final boolean enablePerBlockCompression, final int numericBlockShift, DocOffsetsCodec docOffsetsCodec + ) { + this( + codecName, + skipIndexIntervalSize, + minDocsPerOrdinalForRangeEncoding, + enableOptimizedMerge, + binaryDVCompressionMode, + enablePerBlockCompression, + numericBlockShift, + docOffsetsCodec, + BINARY_DV_BLOCK_BYTES_THRESHOLD_DEFAULT, + BINARY_DV_BLOCK_COUNT_THRESHOLD_DEFAULT + ); + } + + public ES819TSDBDocValuesFormat( + String codecName, + int skipIndexIntervalSize, + int minDocsPerOrdinalForRangeEncoding, + boolean enableOptimizedMerge, + BinaryDVCompressionMode binaryDVCompressionMode, + final boolean enablePerBlockCompression, + final int numericBlockShift, + DocOffsetsCodec docOffsetsCodec, + int blockBytesThreshold, + int blockCountThreshold ) { super(codecName); assert numericBlockShift == NUMERIC_BLOCK_SHIFT || numericBlockShift == NUMERIC_LARGE_BLOCK_SHIFT : numericBlockShift; @@ -242,6 +270,8 @@ public ES819TSDBDocValuesFormat( this.enablePerBlockCompression = enablePerBlockCompression; this.numericBlockShift = numericBlockShift; this.docOffsetsCodec = docOffsetsCodec; + this.blockBytesThreshold = blockBytesThreshold; + this.blockCountThreshold = blockCountThreshold; } @Override @@ -255,6 +285,8 @@ public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOExcept minDocsPerOrdinalForRangeEncoding, enableOptimizedMerge, numericBlockShift, + blockBytesThreshold, + blockCountThreshold, DATA_CODEC, DATA_EXTENSION, META_CODEC, diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819Version3TSDBDocValuesFormat.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819Version3TSDBDocValuesFormat.java index 9e4e3c5352141..13de772f42e85 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819Version3TSDBDocValuesFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819Version3TSDBDocValuesFormat.java @@ -25,21 +25,14 @@ public class ES819Version3TSDBDocValuesFormat extends ES819TSDBDocValuesFormat { static final String CODEC_NAME = "ES8193TSDB"; + static final int BINARY_DV_BLOCK_BYTES_THRESHOLD_DEFAULT = 1024 * 1024; + static final int BINARY_DV_BLOCK_COUNT_THRESHOLD_DEFAULT = 32768; public ES819Version3TSDBDocValuesFormat() { - super( - CODEC_NAME, - DEFAULT_SKIP_INDEX_INTERVAL_SIZE, - ORDINAL_RANGE_ENCODING_MIN_DOC_PER_ORDINAL, - OPTIMIZED_MERGE_ENABLE_DEFAULT, - BinaryDVCompressionMode.COMPRESSED_ZSTD_LEVEL_1, - true, - NUMERIC_BLOCK_SHIFT, - DocOffsetsCodec.BITPACKING - ); + this(false, false); } - public ES819Version3TSDBDocValuesFormat(boolean useLargeNumericBlock) { + public ES819Version3TSDBDocValuesFormat(boolean largeNumericBlock, boolean largeBinaryBlock) { super( CODEC_NAME, DEFAULT_SKIP_INDEX_INTERVAL_SIZE, @@ -47,8 +40,10 @@ public ES819Version3TSDBDocValuesFormat(boolean useLargeNumericBlock) { OPTIMIZED_MERGE_ENABLE_DEFAULT, BinaryDVCompressionMode.COMPRESSED_ZSTD_LEVEL_1, true, - useLargeNumericBlock ? NUMERIC_LARGE_BLOCK_SHIFT : NUMERIC_BLOCK_SHIFT, - DocOffsetsCodec.BITPACKING + largeNumericBlock ? NUMERIC_LARGE_BLOCK_SHIFT : NUMERIC_BLOCK_SHIFT, + DocOffsetsCodec.BITPACKING, + largeBinaryBlock ? BINARY_DV_BLOCK_BYTES_THRESHOLD_DEFAULT : ES819TSDBDocValuesFormat.BINARY_DV_BLOCK_BYTES_THRESHOLD_DEFAULT, + largeBinaryBlock ? BINARY_DV_BLOCK_COUNT_THRESHOLD_DEFAULT : ES819TSDBDocValuesFormat.BINARY_DV_BLOCK_COUNT_THRESHOLD_DEFAULT ); } @@ -68,7 +63,9 @@ public ES819Version3TSDBDocValuesFormat( binaryDVCompressionMode, enablePerBlockCompression, numericBlockShift, - DocOffsetsCodec.BITPACKING + DocOffsetsCodec.BITPACKING, + BINARY_DV_BLOCK_BYTES_THRESHOLD_DEFAULT, + BINARY_DV_BLOCK_COUNT_THRESHOLD_DEFAULT ); } } diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/TSDBDocValuesFormatFactory.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/TSDBDocValuesFormatFactory.java index 6f56f3b4eb6d7..49ea15629120f 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/TSDBDocValuesFormatFactory.java +++ b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/TSDBDocValuesFormatFactory.java @@ -23,7 +23,8 @@ public final class TSDBDocValuesFormatFactory { static final DocValuesFormat ES_819_2_TSDB_DOC_VALUES_FORMAT_LARGE_NUMERIC_BLOCK = ES819TSDBDocValuesFormat.getInstance(true); static final DocValuesFormat ES_819_3_TSDB_DOC_VALUES_FORMAT = new ES819Version3TSDBDocValuesFormat(); - static final DocValuesFormat ES_819_3_TSDB_DOC_VALUES_FORMAT_LARGE_NUMERIC_BLOCK = new ES819Version3TSDBDocValuesFormat(true); + static final DocValuesFormat ES_819_3_TSDB_DOC_VALUES_FORMAT_LARGE_BINARY_BLOCK = new ES819Version3TSDBDocValuesFormat(false, true); + static final DocValuesFormat ES_819_3_TSDB_DOC_VALUES_FORMAT_LARGE_NUMERIC_BLOCK = new ES819Version3TSDBDocValuesFormat(true, false); private TSDBDocValuesFormatFactory() {} @@ -31,16 +32,26 @@ private TSDBDocValuesFormatFactory() {} * Creates and returns a DocValuesFormat instance based on the specified index version * and whether to use a large numeric block size. * - * @param indexCreatedVersion the version of the index being created, which determines - * the applicable DocValuesFormat version. - * @param useLargeBlockSize a boolean flag indicating whether to use a large numeric block size. + * @param indexCreatedVersion the version of the index being created, which determines + * the applicable DocValuesFormat version. + * @param useLargeNumericBlockSize a boolean flag indicating whether to use a large numeric block size. + * @param useLargeBinaryBlockSize a boolean flag indicating whether to use a large binary block size. * @return the appropriate DocValuesFormat instance based on the index version and block size selection. */ - public static DocValuesFormat createDocValuesFormat(IndexVersion indexCreatedVersion, boolean useLargeBlockSize) { + public static DocValuesFormat createDocValuesFormat( + IndexVersion indexCreatedVersion, + boolean useLargeNumericBlockSize, + boolean useLargeBinaryBlockSize + ) { if (indexCreatedVersion.onOrAfter(IndexVersions.TIME_SERIES_DOC_VALUES_FORMAT_VERSION_3)) { - return useLargeBlockSize ? ES_819_3_TSDB_DOC_VALUES_FORMAT_LARGE_NUMERIC_BLOCK : ES_819_3_TSDB_DOC_VALUES_FORMAT; + if (useLargeBinaryBlockSize) { + // At this stage, we don't need large numeric blocks if large binary block is requested: + assert useLargeNumericBlockSize == false; + return ES_819_3_TSDB_DOC_VALUES_FORMAT_LARGE_BINARY_BLOCK; + } + return useLargeNumericBlockSize ? ES_819_3_TSDB_DOC_VALUES_FORMAT_LARGE_NUMERIC_BLOCK : ES_819_3_TSDB_DOC_VALUES_FORMAT; } else { - return useLargeBlockSize ? ES_819_2_TSDB_DOC_VALUES_FORMAT_LARGE_NUMERIC_BLOCK : ES_819_2_TSDB_DOC_VALUES_FORMAT; + return useLargeNumericBlockSize ? ES_819_2_TSDB_DOC_VALUES_FORMAT_LARGE_NUMERIC_BLOCK : ES_819_2_TSDB_DOC_VALUES_FORMAT; } } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java index 873932079b4f8..b9e18ee6911ab 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java @@ -71,8 +71,8 @@ import java.util.function.Supplier; import java.util.stream.IntStream; -import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.BLOCK_BYTES_THRESHOLD; -import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.BLOCK_COUNT_THRESHOLD; +import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.BINARY_DV_BLOCK_BYTES_THRESHOLD_DEFAULT; +import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.BINARY_DV_BLOCK_COUNT_THRESHOLD_DEFAULT; import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SHIFT; import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat.NUMERIC_LARGE_BLOCK_SHIFT; import static org.elasticsearch.test.ESTestCase.between; @@ -143,7 +143,7 @@ public void testBlockWiseBinary() throws Exception { boolean sparse = randomBoolean(); int numBlocksBound = 10; // Since average size is 25b will hit count threshold rather than size threshold, so use count threshold compute needed docs. - int numNonNullValues = randomIntBetween(0, numBlocksBound * BLOCK_COUNT_THRESHOLD); + int numNonNullValues = randomIntBetween(0, numBlocksBound * BINARY_DV_BLOCK_COUNT_THRESHOLD_DEFAULT); List binaryValues = new ArrayList<>(); int numNonNull = 0; @@ -164,7 +164,7 @@ public void testBlockWiseBinary() throws Exception { public void testBlockWiseBinarySmallValues() throws Exception { boolean sparse = randomBoolean(); int numBlocksBound = 5; - int numNonNullValues = randomIntBetween(0, numBlocksBound * BLOCK_COUNT_THRESHOLD); + int numNonNullValues = randomIntBetween(0, numBlocksBound * BINARY_DV_BLOCK_COUNT_THRESHOLD_DEFAULT); List binaryValues = new ArrayList<>(); int numNonNull = 0; @@ -186,7 +186,7 @@ public void testBlockWiseBinaryWithEmptySequences() throws Exception { List binaryValues = new ArrayList<>(); int numSequences = 10; for (int i = 0; i < numSequences; i++) { - int numInSequence = randomIntBetween(0, 3 * BLOCK_COUNT_THRESHOLD); + int numInSequence = randomIntBetween(0, 3 * BINARY_DV_BLOCK_COUNT_THRESHOLD_DEFAULT); boolean emptySequence = randomBoolean(); for (int j = 0; j < numInSequence; j++) { binaryValues.add(emptySequence ? "" : randomAlphaOfLengthBetween(0, 5)); @@ -198,14 +198,17 @@ public void testBlockWiseBinaryWithEmptySequences() throws Exception { public void testBlockWiseBinaryLargeValues() throws Exception { boolean sparse = randomBoolean(); int numBlocksBound = 5; - int binaryDataSize = randomIntBetween(0, numBlocksBound * BLOCK_BYTES_THRESHOLD); + int binaryDataSize = randomIntBetween(0, numBlocksBound * BINARY_DV_BLOCK_BYTES_THRESHOLD_DEFAULT); List binaryValues = new ArrayList<>(); int totalSize = 0; while (totalSize < binaryDataSize) { if (sparse && randomBoolean()) { binaryValues.add(null); } else { - final String value = randomAlphaOfLengthBetween(BLOCK_BYTES_THRESHOLD / 2, 2 * BLOCK_BYTES_THRESHOLD); + final String value = randomAlphaOfLengthBetween( + BINARY_DV_BLOCK_BYTES_THRESHOLD_DEFAULT / 2, + 2 * BINARY_DV_BLOCK_BYTES_THRESHOLD_DEFAULT + ); binaryValues.add(value); totalSize += value.length(); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/TSDBDocValuesFormatFactoryTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/TSDBDocValuesFormatFactoryTests.java index 06149aea13002..167879b2ff5c8 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/TSDBDocValuesFormatFactoryTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/TSDBDocValuesFormatFactoryTests.java @@ -14,46 +14,59 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.index.IndexVersionUtils; +import static org.hamcrest.Matchers.equalTo; + public class TSDBDocValuesFormatFactoryTests extends ESTestCase { - public void testVersion3WithoutLargeBlockSize() { + public void testVersion3() { assertSame( TSDBDocValuesFormatFactory.ES_819_3_TSDB_DOC_VALUES_FORMAT, - TSDBDocValuesFormatFactory.createDocValuesFormat(IndexVersions.TIME_SERIES_DOC_VALUES_FORMAT_VERSION_3, false) + TSDBDocValuesFormatFactory.createDocValuesFormat(IndexVersions.TIME_SERIES_DOC_VALUES_FORMAT_VERSION_3, false, false) ); } - public void testVersion3WithLargeBlockSize() { + public void testVersion3WithLargeNumericBlockSize() { assertSame( TSDBDocValuesFormatFactory.ES_819_3_TSDB_DOC_VALUES_FORMAT_LARGE_NUMERIC_BLOCK, - TSDBDocValuesFormatFactory.createDocValuesFormat(IndexVersions.TIME_SERIES_DOC_VALUES_FORMAT_VERSION_3, true) + TSDBDocValuesFormatFactory.createDocValuesFormat(IndexVersions.TIME_SERIES_DOC_VALUES_FORMAT_VERSION_3, true, false) + ); + } + + public void testVersion3WithLargeBinaryBlockSize() { + var actual = (ES819Version3TSDBDocValuesFormat) TSDBDocValuesFormatFactory.createDocValuesFormat( + IndexVersions.TIME_SERIES_DOC_VALUES_FORMAT_VERSION_3, + false, + true ); + assertSame(TSDBDocValuesFormatFactory.ES_819_3_TSDB_DOC_VALUES_FORMAT_LARGE_BINARY_BLOCK, actual); + assertThat(actual.blockBytesThreshold, equalTo(1024 * 1024)); + assertThat(actual.blockCountThreshold, equalTo(32768)); } public void testVersionAfterVersion3() { assertSame( TSDBDocValuesFormatFactory.ES_819_3_TSDB_DOC_VALUES_FORMAT, - TSDBDocValuesFormatFactory.createDocValuesFormat(IndexVersion.current(), false) + TSDBDocValuesFormatFactory.createDocValuesFormat(IndexVersion.current(), false, false) ); assertSame( TSDBDocValuesFormatFactory.ES_819_3_TSDB_DOC_VALUES_FORMAT_LARGE_NUMERIC_BLOCK, - TSDBDocValuesFormatFactory.createDocValuesFormat(IndexVersion.current(), true) + TSDBDocValuesFormatFactory.createDocValuesFormat(IndexVersion.current(), true, false) ); } - public void testPreVersion3WithoutLargeBlockSize() { + public void testPreVersion3() { IndexVersion preV3 = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_DOC_VALUES_FORMAT_VERSION_3); assertSame( TSDBDocValuesFormatFactory.ES_819_2_TSDB_DOC_VALUES_FORMAT, - TSDBDocValuesFormatFactory.createDocValuesFormat(preV3, false) + TSDBDocValuesFormatFactory.createDocValuesFormat(preV3, false, false) ); } - public void testPreVersion3WithLargeBlockSize() { + public void testPreVersion3WithLargeNumericBlockSize() { IndexVersion preV3 = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_DOC_VALUES_FORMAT_VERSION_3); assertSame( TSDBDocValuesFormatFactory.ES_819_2_TSDB_DOC_VALUES_FORMAT_LARGE_NUMERIC_BLOCK, - TSDBDocValuesFormatFactory.createDocValuesFormat(preV3, true) + TSDBDocValuesFormatFactory.createDocValuesFormat(preV3, true, false) ); } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/TSDBDocValuesFormatSingleNodeTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/TSDBDocValuesFormatSingleNodeTests.java index 5ca0c571fd51c..578f38ed2443a 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/TSDBDocValuesFormatSingleNodeTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/TSDBDocValuesFormatSingleNodeTests.java @@ -71,6 +71,21 @@ public void testStandardIndexWithTSDBDocValuesFormatSetting() throws Exception { assertDocValuesFormat(indexName, TSDBDocValuesFormatFactory.ES_819_3_TSDB_DOC_VALUES_FORMAT, expectedFields); } + public void testuUeTimeSeriesDocValuesFormatLargeBinaryBlockSize() throws Exception { + String indexName = "standard-larger-binary-db-block-size-dv-test"; + Settings settings = Settings.builder() + .put(IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING.getKey(), true) + .put(IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BINARY_BLOCK_SIZE.getKey(), true) + .build(); + + createIndex(indexName, settings, "@timestamp", "type=date", "hostname", "type=keyword", "gauge", "type=long"); + + indexDocuments(indexName); + + Set expectedFields = Set.of("@timestamp", "hostname", "gauge", "_seq_no"); + assertDocValuesFormat(indexName, TSDBDocValuesFormatFactory.ES_819_3_TSDB_DOC_VALUES_FORMAT_LARGE_BINARY_BLOCK, expectedFields); + } + private void indexDocuments(String indexName) throws Exception { long baseTimestamp = 1704067200000L; int numDocs = randomIntBetween(5, 20);