From 633d283d4a2c72560bec11cf176ee7801cb53189 Mon Sep 17 00:00:00 2001 From: Kaustubh Butte Date: Wed, 11 Feb 2026 11:00:57 +0530 Subject: [PATCH 01/12] Add warmup phase for pull-based ingestion to prevent serving stale data This PR introduces a warmup phase for pull-based ingestion that prevents shards from serving queries until they have caught up with the streaming source. This is analogous to how push-based replication waits for replicas to sync before serving. Key changes: - Add warmup settings: enabled (default: true), timeout, lag_threshold, fail_on_timeout - Create WarmupConfig class to encapsulate warmup configuration - Block shard in postRecovery() until warmup completes or times out - Use CountDownLatch for thread-safe warmup blocking/signaling - Encapsulate timeout and error handling in IngestionEngine.awaitWarmupComplete() New index settings: - index.ingestion_source.warmup.enabled (boolean, default: true) - index.ingestion_source.warmup.timeout (time, default: 5m) - index.ingestion_source.warmup.lag_threshold (long, default: 0) - index.ingestion_source.warmup.fail_on_timeout (boolean, default: false) Resolves #20506 Signed-off-by: Kaustubh Butte --- CHANGELOG.md | 7 + .../plugin/kafka/IngestFromKafkaIT.java | 13 + .../plugin/kafka/KafkaIngestionBaseIT.java | 1 + .../plugin/kafka/RemoteStoreKafkaIT.java | 8 + .../plugin/kafka/KafkaSingleNodeTests.java | 2 + .../cluster/metadata/IndexMetadata.java | 57 +++ .../cluster/metadata/IngestionSource.java | 147 ++++++- .../common/settings/IndexScopedSettings.java | 4 + .../index/engine/IngestionEngine.java | 53 +++ .../opensearch/index/shard/IndexShard.java | 13 + .../pollingingest/DefaultStreamPoller.java | 163 +++++++- .../indices/pollingingest/StreamPoller.java | 14 + .../metadata/IngestionSourceTests.java | 108 ++++- .../DefaultStreamPollerTests.java | 385 +++++++++++++++++- 14 files changed, 954 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 276ba96567449..77c327b2bbba1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) - Add ref_path support for package-based hunspell dictionary loading ([#20840](https://github.com/opensearch-project/OpenSearch/pull/20840)) - Add support for enabling pluggable data formats, starting with phase-1 of decoupling shard from engine, and introducing basic abstractions ([#20675](https://github.com/opensearch-project/OpenSearch/pull/20675)) +- Add getWrappedScorer method to ProfileScorer for plugin access to wrapped scorers ([#20548](https://github.com/opensearch-project/OpenSearch/issues/20548)) +- Support expected cluster name with validation in CCS Sniff mode ([#20532](https://github.com/opensearch-project/OpenSearch/pull/20532)) +- Add security policy to allow `accessUnixDomainSocket` in `transport-grpc` module ([#20463](https://github.com/opensearch-project/OpenSearch/pull/20463)) +- [Workload Management] Enhance Scroll API support for autotagging ([#20151](https://github.com/opensearch-project/OpenSearch/pull/20151)) +- Add warmup phase to wait for lag to catch up in pull-based ingestion before serving ([#20526](https://github.com/opensearch-project/OpenSearch/pull/20526)) ### Changed - Make telemetry `Tags` immutable ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788)) @@ -58,6 +63,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Delegate getMin/getMax methods for ExitableTerms ([#20775](https://github.com/opensearch-project/OpenSearch/pull/20775)) - Fix terms lookup subquery fetch limit reading from non-existent index setting instead of cluster `max_clause_count` ([#20823](https://github.com/opensearch-project/OpenSearch/pull/20823)) - Fix array_index_out_of_bounds_exception with wildcard and aggregations ([#20842](https://github.com/opensearch-project/OpenSearch/pull/20842)) +- - Delegate getMin/getMax methods for ExitableTerms ([#20775](https://github.com/opensearch-project/OpenSearch/pull/20775)) +- Fix CriteriaBasedCodec to work with delegate codec. ([20442](https://github.com/opensearch-project/OpenSearch/pull/20442)) ### Dependencies - Bump shadow-gradle-plugin from 8.3.9 to 9.3.1 ([#20569](https://github.com/opensearch-project/OpenSearch/pull/20569)) diff --git a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java index 814021ddf0b32..af863d6519180 100644 --- a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java +++ b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java @@ -107,6 +107,7 @@ public void testKafkaIngestion_RewindByTimeStamp() { .put("ingestion_source.param.topic", "test") .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "latest") + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -135,6 +136,7 @@ public void testKafkaIngestion_RewindByOffset() { .put("ingestion_source.param.topic", "test") .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "latest") + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -270,6 +272,7 @@ public void testAllActiveIngestion() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.pointer.init.reset", "earliest") + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -369,6 +372,7 @@ public void testReplicaPromotionOnAllActiveIngestion() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.pointer.init.reset", "earliest") + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -421,6 +425,7 @@ public void testSnapshotRestoreOnAllActiveIngestion() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.pointer.init.reset", "earliest") + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -502,6 +507,7 @@ public void testResetPollerInAllActiveIngestion() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.pointer.init.reset", "earliest") + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -542,6 +548,7 @@ public void testAllActiveOffsetBasedLag() throws Exception { .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.pointer_based_lag_update_interval", "3s") + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -613,6 +620,7 @@ public void testAllActiveIngestionBatchStartPointerOnReplicaPromotion() throws E .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -707,6 +715,7 @@ public void testAllActiveIngestionPeriodicFlush() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .put("index.periodic_flush_interval", "5s") .build(), @@ -742,6 +751,7 @@ public void testRawPayloadMapperIngestion() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.mapper_type", "raw_payload") .put("ingestion_source.error_strategy", "drop") + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -859,6 +869,7 @@ public void testDynamicUpdateKafkaParams() throws Exception { .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "latest") .put("ingestion_source.param.max.poll.records", "100") + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), mapping @@ -969,6 +980,7 @@ public void testConsumerInitializationFailureAndRecovery() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "none") + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), mapping @@ -1049,6 +1061,7 @@ public void testDynamicConfigUpdateOnNoMessages() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "earliest") + .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), mapping diff --git a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/KafkaIngestionBaseIT.java b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/KafkaIngestionBaseIT.java index 604df7a7c8414..7c57c74e425ee 100644 --- a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/KafkaIngestionBaseIT.java +++ b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/KafkaIngestionBaseIT.java @@ -205,6 +205,7 @@ protected void createIndexWithDefaultSettings(String indexName, int numShards, i // set custom kafka consumer properties .put("ingestion_source.param.fetch.min.bytes", 30000) .put("ingestion_source.param.enable.auto.commit", false) + .put("ingestion_source.warmup.enabled", false) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" ); diff --git a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/RemoteStoreKafkaIT.java b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/RemoteStoreKafkaIT.java index 1d6f49e473bc0..5c712dd406341 100644 --- a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/RemoteStoreKafkaIT.java +++ b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/RemoteStoreKafkaIT.java @@ -157,6 +157,7 @@ public void testErrorStrategy() throws Exception { .put("ingestion_source.internal_queue_size", "1000") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) + .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -398,6 +399,7 @@ public void testExternalVersioningWithDisabledGCDeletes() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) + .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .put("index.gc_deletes", "0") .build(), @@ -510,6 +512,7 @@ public void testOffsetUpdateOnBlockErrorPolicy() throws Exception { .put("ingestion_source.internal_queue_size", "1000") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) + .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -555,6 +558,7 @@ public void testConsumerResetByTimestamp() throws Exception { .put("ingestion_source.internal_queue_size", "1000") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) + .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -673,6 +677,7 @@ public void testIndexRelocation() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "earliest") + .put("ingestion_source.warmup.enabled", false) .put("index.routing.allocation.require._name", nodeA) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -724,6 +729,7 @@ public void testKafkaConnectionLost() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "earliest") + .put("ingestion_source.warmup.enabled", false) .put("index.routing.allocation.require._name", nodeA) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -787,6 +793,7 @@ public void testBatchStartPointerOnReplicaPromotion() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) + .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -850,6 +857,7 @@ public void testPeriodicFlush() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) + .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .put("index.periodic_flush_interval", "5s") .build(), diff --git a/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java b/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java index 7a14782d0a1a8..428c7df1dde36 100644 --- a/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java +++ b/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java @@ -140,6 +140,7 @@ public void testPauseAndResumeAPIs() throws Exception { } // This test validates shard initialization does not fail due to kafka connection errors. + // Warmup is disabled since we're testing error handling, not warmup behavior. public void testShardInitializationUsingUnknownTopic() throws Exception { createIndexWithMappingSource( indexName, @@ -150,6 +151,7 @@ public void testShardInitializationUsingUnknownTopic() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", "unknownTopic") .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) + .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .build(), mappings diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index e43df0ea35b80..6517a44c9d936 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -1031,6 +1031,54 @@ public Iterator> settings() { }, Property.IndexScope, Property.Final) ); + /** + * Defines if warmup phase is enabled for pull-based ingestion. When enabled, shards will wait for + * lag to catch up before serving queries after node restart or shard relocation. + */ + public static final String SETTING_INGESTION_SOURCE_WARMUP_ENABLED = "index.ingestion_source.warmup.enabled"; + public static final Setting INGESTION_SOURCE_WARMUP_ENABLED_SETTING = Setting.boolSetting( + SETTING_INGESTION_SOURCE_WARMUP_ENABLED, + true, + Property.IndexScope, + Property.Final + ); + + /** + * Defines the maximum time to wait for lag to catch up during warmup phase. + */ + public static final String SETTING_INGESTION_SOURCE_WARMUP_TIMEOUT = "index.ingestion_source.warmup.timeout"; + public static final Setting INGESTION_SOURCE_WARMUP_TIMEOUT_SETTING = Setting.positiveTimeSetting( + SETTING_INGESTION_SOURCE_WARMUP_TIMEOUT, + new TimeValue(5, TimeUnit.MINUTES), + Property.IndexScope, + Property.Final + ); + + /** + * Defines the acceptable pointer-based lag threshold. Warmup completes when lag is at or below this value. + * A value of 0 means fully caught up (no lag). + */ + public static final String SETTING_INGESTION_SOURCE_WARMUP_LAG_THRESHOLD = "index.ingestion_source.warmup.lag_threshold"; + public static final Setting INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING = Setting.longSetting( + SETTING_INGESTION_SOURCE_WARMUP_LAG_THRESHOLD, + 0L, + 0L, + Property.IndexScope, + Property.Final + ); + + /** + * Defines if shard initialization should fail when warmup times out. + * If false, shard proceeds with a warning. If true, shard initialization fails. + */ + public static final String SETTING_INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT = "index.ingestion_source.warmup.fail_on_timeout"; + public static final Setting INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT_SETTING = Setting.boolSetting( + SETTING_INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT, + false, + Property.IndexScope, + Property.Final + ); + /** * an internal index format description, allowing us to find out if this index is upgraded or needs upgrading */ @@ -1301,6 +1349,14 @@ public IngestionSource getIngestionSource() { final IngestionMessageMapper.MapperType mapperType = INGESTION_SOURCE_MAPPER_TYPE_SETTING.get(settings); final Map mapperSettings = INGESTION_SOURCE_MAPPER_SETTINGS.getAsMap(settings); + // Warmup settings + final IngestionSource.WarmupConfig warmupConfig = new IngestionSource.WarmupConfig( + INGESTION_SOURCE_WARMUP_ENABLED_SETTING.get(settings), + INGESTION_SOURCE_WARMUP_TIMEOUT_SETTING.get(settings), + INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING.get(settings), + INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT_SETTING.get(settings) + ); + return new IngestionSource.Builder(ingestionSourceType).setParams(ingestionSourceParams) .setPointerInitReset(pointerInitReset) .setErrorStrategy(errorStrategy) @@ -1312,6 +1368,7 @@ public IngestionSource getIngestionSource() { .setPointerBasedLagUpdateInterval(pointerBasedLagUpdateInterval) .setMapperType(mapperType) .setMapperSettings(mapperSettings) + .setWarmupConfig(warmupConfig) .build(); } return null; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java index ff0bda28cc88e..ff11df9d3ca1d 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java @@ -8,6 +8,7 @@ package org.opensearch.cluster.metadata; +import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -27,6 +28,10 @@ import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_NUM_PROCESSOR_THREADS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_POINTER_BASED_LAG_UPDATE_INTERVAL_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_POLL_TIMEOUT; +import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_WARMUP_ENABLED_SETTING; +import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT_SETTING; +import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING; +import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_WARMUP_TIMEOUT_SETTING; /** * Class encapsulating the configuration of an ingestion source. @@ -45,6 +50,7 @@ public class IngestionSource { private final TimeValue pointerBasedLagUpdateInterval; private final IngestionMessageMapper.MapperType mapperType; private final Map mapperSettings; + private final WarmupConfig warmupConfig; private IngestionSource( String type, @@ -58,7 +64,8 @@ private IngestionSource( boolean allActiveIngestion, TimeValue pointerBasedLagUpdateInterval, IngestionMessageMapper.MapperType mapperType, - Map mapperSettings + Map mapperSettings, + WarmupConfig warmupConfig ) { this.type = type; this.pointerInitReset = pointerInitReset; @@ -72,6 +79,7 @@ private IngestionSource( this.pointerBasedLagUpdateInterval = pointerBasedLagUpdateInterval; this.mapperType = mapperType; this.mapperSettings = mapperSettings != null ? Collections.unmodifiableMap(mapperSettings) : Collections.emptyMap(); + this.warmupConfig = warmupConfig; } public String getType() { @@ -122,6 +130,26 @@ public Map getMapperSettings() { return mapperSettings; } + public WarmupConfig getWarmupConfig() { + return warmupConfig; + } + + public boolean isWarmupEnabled() { + return warmupConfig.isEnabled(); + } + + public TimeValue getWarmupTimeout() { + return warmupConfig.getTimeout(); + } + + public long getWarmupLagThreshold() { + return warmupConfig.getLagThreshold(); + } + + public boolean isWarmupFailOnTimeout() { + return warmupConfig.isFailOnTimeout(); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -138,7 +166,8 @@ public boolean equals(Object o) { && Objects.equals(allActiveIngestion, ingestionSource.allActiveIngestion) && Objects.equals(pointerBasedLagUpdateInterval, ingestionSource.pointerBasedLagUpdateInterval) && Objects.equals(mapperType, ingestionSource.mapperType) - && Objects.equals(mapperSettings, ingestionSource.mapperSettings); + && Objects.equals(mapperSettings, ingestionSource.mapperSettings) + && Objects.equals(warmupConfig, ingestionSource.warmupConfig); } @Override @@ -155,7 +184,8 @@ public int hashCode() { allActiveIngestion, pointerBasedLagUpdateInterval, mapperType, - mapperSettings + mapperSettings, + warmupConfig ); } @@ -190,6 +220,8 @@ public String toString() { + '\'' + ", mapperSettings=" + mapperSettings + + ", warmupConfig=" + + warmupConfig + '}'; } @@ -233,6 +265,72 @@ public String toString() { } } + /** + * Class encapsulating the warmup configuration for pull-based ingestion. + * When warmup is enabled, shards will wait for lag to catch up before serving queries + * after node restart or shard relocation. + */ + @ExperimentalApi + public static class WarmupConfig { + private final boolean enabled; + private final TimeValue timeout; + private final long lagThreshold; + private final boolean failOnTimeout; + + public WarmupConfig(boolean enabled, TimeValue timeout, long lagThreshold, boolean failOnTimeout) { + this.enabled = enabled; + this.timeout = timeout; + this.lagThreshold = lagThreshold; + this.failOnTimeout = failOnTimeout; + } + + public boolean isEnabled() { + return enabled; + } + + public TimeValue getTimeout() { + return timeout; + } + + public long getLagThreshold() { + return lagThreshold; + } + + public boolean isFailOnTimeout() { + return failOnTimeout; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + WarmupConfig that = (WarmupConfig) o; + return enabled == that.enabled + && lagThreshold == that.lagThreshold + && failOnTimeout == that.failOnTimeout + && Objects.equals(timeout, that.timeout); + } + + @Override + public int hashCode() { + return Objects.hash(enabled, timeout, lagThreshold, failOnTimeout); + } + + @Override + public String toString() { + return "WarmupConfig{" + + "enabled=" + + enabled + + ", timeout=" + + timeout + + ", lagThreshold=" + + lagThreshold + + ", failOnTimeout=" + + failOnTimeout + + '}'; + } + } + /** * Builder for {@link IngestionSource}. * @@ -253,6 +351,11 @@ public static class Builder { ); private IngestionMessageMapper.MapperType mapperType = INGESTION_SOURCE_MAPPER_TYPE_SETTING.getDefault(Settings.EMPTY); private Map mapperSettings = new HashMap<>(); + // Warmup configuration + private boolean warmupEnabled = INGESTION_SOURCE_WARMUP_ENABLED_SETTING.getDefault(Settings.EMPTY); + private TimeValue warmupTimeout = INGESTION_SOURCE_WARMUP_TIMEOUT_SETTING.getDefault(Settings.EMPTY); + private long warmupLagThreshold = INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING.getDefault(Settings.EMPTY); + private boolean warmupFailOnTimeout = INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT_SETTING.getDefault(Settings.EMPTY); public Builder(String type) { this.type = type; @@ -269,6 +372,12 @@ public Builder(IngestionSource ingestionSource) { this.pointerBasedLagUpdateInterval = ingestionSource.pointerBasedLagUpdateInterval; this.mapperType = ingestionSource.mapperType; this.mapperSettings = new HashMap<>(ingestionSource.mapperSettings); + // Copy warmup config + WarmupConfig wc = ingestionSource.warmupConfig; + this.warmupEnabled = wc.isEnabled(); + this.warmupTimeout = wc.getTimeout(); + this.warmupLagThreshold = wc.getLagThreshold(); + this.warmupFailOnTimeout = wc.isFailOnTimeout(); } public Builder setPointerInitReset(PointerInitReset pointerInitReset) { @@ -331,7 +440,36 @@ public Builder setMapperSettings(Map mapperSettings) { return this; } + public Builder setWarmupEnabled(boolean warmupEnabled) { + this.warmupEnabled = warmupEnabled; + return this; + } + + public Builder setWarmupTimeout(TimeValue warmupTimeout) { + this.warmupTimeout = warmupTimeout; + return this; + } + + public Builder setWarmupLagThreshold(long warmupLagThreshold) { + this.warmupLagThreshold = warmupLagThreshold; + return this; + } + + public Builder setWarmupFailOnTimeout(boolean warmupFailOnTimeout) { + this.warmupFailOnTimeout = warmupFailOnTimeout; + return this; + } + + public Builder setWarmupConfig(WarmupConfig warmupConfig) { + this.warmupEnabled = warmupConfig.isEnabled(); + this.warmupTimeout = warmupConfig.getTimeout(); + this.warmupLagThreshold = warmupConfig.getLagThreshold(); + this.warmupFailOnTimeout = warmupConfig.isFailOnTimeout(); + return this; + } + public IngestionSource build() { + WarmupConfig warmupConfig = new WarmupConfig(warmupEnabled, warmupTimeout, warmupLagThreshold, warmupFailOnTimeout); return new IngestionSource( type, pointerInitReset, @@ -344,7 +482,8 @@ public IngestionSource build() { allActiveIngestion, pointerBasedLagUpdateInterval, mapperType, - mapperSettings + mapperSettings, + warmupConfig ); } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index ce63a66a4b27c..d589b39fe81b9 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -292,6 +292,10 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexMetadata.INGESTION_SOURCE_POINTER_BASED_LAG_UPDATE_INTERVAL_SETTING, IndexMetadata.INGESTION_SOURCE_MAPPER_TYPE_SETTING, IndexMetadata.INGESTION_SOURCE_MAPPER_SETTINGS, + IndexMetadata.INGESTION_SOURCE_WARMUP_ENABLED_SETTING, + IndexMetadata.INGESTION_SOURCE_WARMUP_TIMEOUT_SETTING, + IndexMetadata.INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING, + IndexMetadata.INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT_SETTING, // Settings for search replica IndexMetadata.INDEX_NUMBER_OF_SEARCH_REPLICAS_SETTING, diff --git a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java index ef6df80f2a47f..ffeabae2fd119 100644 --- a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java @@ -155,6 +155,10 @@ private void initializeStreamPoller( .pointerBasedLagUpdateInterval(ingestionSource.getPointerBasedLagUpdateInterval().millis()) .mapperType(ingestionSource.getMapperType()) .mapperSettings(ingestionSource.getMapperSettings()) + .warmupEnabled(ingestionSource.isWarmupEnabled()) + .warmupTimeoutMs(ingestionSource.getWarmupTimeout().millis()) + .warmupLagThreshold(ingestionSource.getWarmupLagThreshold()) + .warmupFailOnTimeout(ingestionSource.isWarmupFailOnTimeout()) .build(); registerStreamPollerListener(); @@ -662,4 +666,53 @@ public ShardIngestionState getIngestionState() { shardPointer != null ? shardPointer.toString() : "" ); } + + /** + * Returns true if the warmup phase is complete and the shard is ready to serve. + */ + public boolean isWarmupComplete() { + return streamPoller.isWarmupComplete(); + } + + /** + * Block until warmup is complete or timeout occurs. + * This method handles all warmup logic internally including timeout handling and error throwing. + * + * @throws OpenSearchException if warmup times out and failOnTimeout is configured + * @throws InterruptedException if the thread is interrupted while waiting + */ + public void awaitWarmupComplete() throws InterruptedException { + IngestionSource ingestionSource = engineConfig.getIndexSettings().getIndexMetadata().getIngestionSource(); + if (ingestionSource == null || !ingestionSource.isWarmupEnabled()) { + return; + } + + long timeoutMs = ingestionSource.getWarmupTimeout().millis(); + boolean completed = streamPoller.awaitWarmupComplete(timeoutMs); + + if (!completed) { + if (isWarmupFailOnTimeout()) { + throw new OpenSearchException( + "Ingestion warmup timed out for shard after " + + timeoutMs + + "ms. " + + "Configure warmup.fail_on_timeout=false to proceed with stale data." + ); + } + // Log warning when proceeding despite timeout + logger.warn( + "Ingestion warmup timed out for shard after {}ms, proceeding with potentially stale data. " + + "Set warmup.fail_on_timeout=true to fail shard initialization on timeout.", + timeoutMs + ); + } + } + + /** + * Returns true if shard initialization should fail when warmup times out. + */ + public boolean isWarmupFailOnTimeout() { + IngestionSource ingestionSource = engineConfig.getIndexSettings().getIndexMetadata().getIngestionSource(); + return ingestionSource != null && ingestionSource.isWarmupFailOnTimeout(); + } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index a15dd0b1030b8..2e58b7224bede 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -2487,6 +2487,19 @@ public void postRecovery(String reason) throws IndexShardStartedException, Index // responded to in addRefreshListener. The refresh must happen under the same mutex used in addRefreshListener // and before moving this shard to POST_RECOVERY state (i.e., allow to read from this shard). getIndexer().refresh("post_recovery"); + + // Wait for ingestion warmup if enabled (pull-based ingestion only) + Engine engine = getEngine(); + if (engine instanceof IngestionEngine) { + IngestionEngine ingestionEngine = (IngestionEngine) engine; + try { + ingestionEngine.awaitWarmupComplete(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new OpenSearchException("Interrupted waiting for ingestion warmup", e); + } + } + synchronized (mutex) { if (state == IndexShardState.CLOSED) { throw new IndexShardClosedException(shardId); diff --git a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java index 2571f7b7ac29a..8c8ea6cd9fd7d 100644 --- a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java +++ b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java @@ -30,8 +30,10 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; /** * Default implementation of {@link StreamPoller} @@ -59,6 +61,15 @@ public class DefaultStreamPoller implements StreamPoller { private volatile long cachedPointerBasedLag = 0; private volatile long lastPointerBasedLagUpdateTime = 0; + // Warmup configuration and state + private final boolean warmupEnabled; + private final long warmupTimeoutMs; + private final long warmupLagThreshold; + private final boolean warmupFailOnTimeout; + private volatile boolean warmupComplete = false; + private volatile long warmupStartTime = 0; + private final CountDownLatch warmupLatch = new CountDownLatch(1); + @Nullable private IngestionShardConsumer consumer; private IngestionConsumerFactory consumerFactory; @@ -107,7 +118,11 @@ private DefaultStreamPoller( int blockingQueueSize, long pointerBasedLagUpdateIntervalMs, IngestionMessageMapper.MapperType mapperType, - Map mapperSettings + Map mapperSettings, + boolean warmupEnabled, + long warmupTimeoutMs, + long warmupLagThreshold, + boolean warmupFailOnTimeout ) { this( startPointer, @@ -123,7 +138,11 @@ private DefaultStreamPoller( pollTimeout, pointerBasedLagUpdateIntervalMs, ingestionEngine.config().getIndexSettings(), - IngestionMessageMapper.create(mapperType.getName(), shardId, mapperSettings) + IngestionMessageMapper.create(mapperType.getName(), shardId, mapperSettings), + warmupEnabled, + warmupTimeoutMs, + warmupLagThreshold, + warmupFailOnTimeout ); } @@ -144,7 +163,11 @@ private DefaultStreamPoller( int pollTimeout, long pointerBasedLagUpdateIntervalMs, IndexSettings indexSettings, - IngestionMessageMapper messageMapper + IngestionMessageMapper messageMapper, + boolean warmupEnabled, + long warmupTimeoutMs, + long warmupLagThreshold, + boolean warmupFailOnTimeout ) { this.consumerFactory = Objects.requireNonNull(consumerFactory); this.consumerClientId = Objects.requireNonNull(consumerClientId); @@ -163,9 +186,17 @@ private DefaultStreamPoller( this.errorStrategy = errorStrategy; this.indexName = indexSettings.getIndex().getName(); this.messageMapper = Objects.requireNonNull(messageMapper); + this.warmupEnabled = warmupEnabled; + this.warmupTimeoutMs = warmupTimeoutMs; + this.warmupLagThreshold = warmupLagThreshold; + this.warmupFailOnTimeout = warmupFailOnTimeout; // handle initial poller states this.paused = initialState == State.PAUSED; + // If warmup is disabled, mark as complete immediately + if (!warmupEnabled) { + this.warmupComplete = true; + } } @Override @@ -195,6 +226,13 @@ protected void startPoll() { } logger.info("Starting poller for shard {}", shardId); + // Initialize warmup if enabled + if (warmupEnabled && !warmupComplete) { + warmupStartTime = System.currentTimeMillis(); + state = State.WARMING_UP; + logger.info("Starting warmup phase for index {} shard {}, waiting for lag to catch up", indexName, shardId); + } + while (true) { try { if (closed) { @@ -212,6 +250,11 @@ protected void startPoll() { // Update lag periodically. Lag is updated even if the poller is paused. updatePointerBasedLagIfNeeded(); + // Check warmup status if not yet complete + if (!warmupComplete && warmupEnabled) { + checkWarmupStatus(); + } + if (paused || isWriteBlockEnabled) { state = State.PAUSED; try { @@ -355,6 +398,77 @@ public boolean isClosed() { return closed; } + @Override + public boolean isWarmupComplete() { + return warmupComplete || !warmupEnabled; + } + + /** + * Returns true if shard initialization should fail when warmup times out. + */ + public boolean isWarmupFailOnTimeout() { + return warmupFailOnTimeout; + } + + @Override + public boolean awaitWarmupComplete(long timeoutMs) throws InterruptedException { + if (!warmupEnabled || isWarmupComplete()) { + return true; + } + + boolean completed = warmupLatch.await(timeoutMs, TimeUnit.MILLISECONDS); + if (!completed) { + logger.warn( + "Warmup timeout for index {} shard {} - proceeding with current lag (warmupComplete={})", + indexName, + shardId, + isWarmupComplete() + ); + } + return completed; + } + + /** + * Check if warmup conditions are met and mark warmup as complete if so. + * + * Warmup uses offset-based lag (cachedPointerBasedLag) which tracks the difference between + * the current consumer position and the end of the stream. This is the preferred mode for + * Kafka and other sources that support offset-based lag calculation. + * Note: cachedPointerBasedLag is 0 by default and is only updated after updatePointerBasedLagIfNeeded() + * is called. + */ + private void checkWarmupStatus() { + long currentLag = cachedPointerBasedLag; + long threshold = warmupLagThreshold; + + long elapsedTime = System.currentTimeMillis() - warmupStartTime; + boolean lagBelowThreshold = currentLag >= 0 && currentLag <= threshold; + boolean timeoutReached = elapsedTime >= warmupTimeoutMs; + + if (lagBelowThreshold) { + warmupComplete = true; + warmupLatch.countDown(); + logger.info( + "Warmup complete for index {} shard {} - lag {} is at or below threshold {}", + indexName, + shardId, + currentLag, + threshold + ); + } else if (timeoutReached) { + warmupComplete = true; + warmupLatch.countDown(); + logger.warn( + "Warmup timeout for index {} shard {} after {}ms - proceeding with lag {} (threshold was {})", + indexName, + shardId, + elapsedTime, + currentLag, + threshold + ); + } + } + /** * Returns the batch start pointer from where the poller can resume in case of shard recovery. The poller and * processor are decoupled in this implementation, and hence the latest pointer tracked by the processor acts as the @@ -608,6 +722,11 @@ public static class Builder { private long pointerBasedLagUpdateIntervalMs = 10000; private IngestionMessageMapper.MapperType mapperType = IngestionMessageMapper.MapperType.DEFAULT; private Map mapperSettings = Collections.emptyMap(); + // Warmup configuration - defaults match IndexMetadata settings + private boolean warmupEnabled = true; + private long warmupTimeoutMs = 300000; // 5 minutes + private long warmupLagThreshold = 0; + private boolean warmupFailOnTimeout = false; /** * Initialize the builder with mandatory parameters @@ -715,6 +834,38 @@ public Builder mapperSettings(Map mapperSettings) { return this; } + /** + * Set warmup enabled + */ + public Builder warmupEnabled(boolean warmupEnabled) { + this.warmupEnabled = warmupEnabled; + return this; + } + + /** + * Set warmup timeout in milliseconds + */ + public Builder warmupTimeoutMs(long warmupTimeoutMs) { + this.warmupTimeoutMs = warmupTimeoutMs; + return this; + } + + /** + * Set warmup lag threshold (pointer-based) + */ + public Builder warmupLagThreshold(long warmupLagThreshold) { + this.warmupLagThreshold = warmupLagThreshold; + return this; + } + + /** + * Set whether shard initialization should fail on warmup timeout + */ + public Builder warmupFailOnTimeout(boolean warmupFailOnTimeout) { + this.warmupFailOnTimeout = warmupFailOnTimeout; + return this; + } + /** * Build the DefaultStreamPoller instance */ @@ -735,7 +886,11 @@ public DefaultStreamPoller build() { blockingQueueSize, pointerBasedLagUpdateIntervalMs, mapperType, - mapperSettings + mapperSettings, + warmupEnabled, + warmupTimeoutMs, + warmupLagThreshold, + warmupFailOnTimeout ); } } diff --git a/server/src/main/java/org/opensearch/indices/pollingingest/StreamPoller.java b/server/src/main/java/org/opensearch/indices/pollingingest/StreamPoller.java index dee5bd3384389..1d9249cb3f57c 100644 --- a/server/src/main/java/org/opensearch/indices/pollingingest/StreamPoller.java +++ b/server/src/main/java/org/opensearch/indices/pollingingest/StreamPoller.java @@ -83,11 +83,25 @@ public interface StreamPoller extends Closeable, ClusterStateListener { */ void requestConsumerReinitialization(IngestionSource updatedIngestionSource); + /** + * @return true if the warmup phase is complete and the shard is ready to serve + */ + boolean isWarmupComplete(); + + /** + * Block until warmup is complete or timeout occurs. + * @param timeoutMs maximum time to wait in milliseconds + * @return true if warmup completed, false if timeout + * @throws InterruptedException if the thread is interrupted while waiting + */ + boolean awaitWarmupComplete(long timeoutMs) throws InterruptedException; + /** * A state to indicate the current state of the poller */ enum State { NONE, + WARMING_UP, CLOSED, PAUSED, POLLING, diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java index 1c3f74ead393e..80a66465cebfe 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java @@ -108,7 +108,7 @@ public void testToString() { .setErrorStrategy(DROP) .build(); String expected = - "IngestionSource{type='type',pointer_init_reset='PointerInitReset{type='RESET_BY_OFFSET', value=1000}',error_strategy='DROP', params={key=value}, maxPollSize=1000, pollTimeout=1000, numProcessorThreads=1, blockingQueueSize=100, allActiveIngestion=false, pointerBasedLagUpdateInterval=10s, mapperType='DEFAULT', mapperSettings={}}"; + "IngestionSource{type='type',pointer_init_reset='PointerInitReset{type='RESET_BY_OFFSET', value=1000}',error_strategy='DROP', params={key=value}, maxPollSize=1000, pollTimeout=1000, numProcessorThreads=1, blockingQueueSize=100, allActiveIngestion=false, pointerBasedLagUpdateInterval=10s, mapperType='DEFAULT', mapperSettings={}, warmupConfig=WarmupConfig{enabled=true, timeout=5m, lagThreshold=0, failOnTimeout=false}}"; assertEquals(expected, source.toString()); } @@ -177,4 +177,110 @@ public void testMapperSettings() { .build(); assertTrue(sourceNoMapperSettings.getMapperSettings().isEmpty()); } + + public void testWarmupConfigurationConstructorAndGetters() { + Map params = new HashMap<>(); + params.put("key", "value"); + + // Test with warmup configuration + IngestionSource source = new IngestionSource.Builder("type").setParams(params) + .setPointerInitReset(pointerInitReset) + .setErrorStrategy(DROP) + .setWarmupEnabled(true) + .setWarmupTimeout(TimeValue.timeValueMinutes(10)) + .setWarmupLagThreshold(100) + .setWarmupFailOnTimeout(true) + .build(); + + assertTrue("Warmup should be enabled", source.isWarmupEnabled()); + assertEquals(TimeValue.timeValueMinutes(10), source.getWarmupTimeout()); + assertEquals(100, source.getWarmupLagThreshold()); + assertTrue("Should fail on timeout", source.isWarmupFailOnTimeout()); + } + + public void testWarmupConfigurationDefaults() { + // Test default warmup values + IngestionSource source = new IngestionSource.Builder("type").build(); + + assertTrue("Warmup should be enabled by default", source.isWarmupEnabled()); + assertEquals(TimeValue.timeValueMinutes(5), source.getWarmupTimeout()); + assertEquals(0, source.getWarmupLagThreshold()); + assertFalse("Should not fail on timeout by default", source.isWarmupFailOnTimeout()); + } + + public void testWarmupConfigurationEquality() { + Map params = new HashMap<>(); + params.put("key", "value"); + + IngestionSource source1 = new IngestionSource.Builder("type").setParams(params) + .setWarmupEnabled(true) + .setWarmupLagThreshold(100) + .build(); + + IngestionSource source2 = new IngestionSource.Builder("type").setParams(params) + .setWarmupEnabled(true) + .setWarmupLagThreshold(100) + .build(); + + assertEquals(source1, source2); + assertEquals(source1.hashCode(), source2.hashCode()); + + // Test inequality with different warmup settings + IngestionSource source3 = new IngestionSource.Builder("type").setParams(params) + .setWarmupEnabled(false) + .setWarmupLagThreshold(100) + .build(); + + assertNotEquals(source1, source3); + } + + public void testWarmupConfigurationCopiedByBuilder() { + IngestionSource original = new IngestionSource.Builder("type").setWarmupEnabled(true) + .setWarmupTimeout(TimeValue.timeValueMinutes(10)) + .setWarmupLagThreshold(500) + .setWarmupFailOnTimeout(true) + .build(); + + // Create a copy using the copy constructor + IngestionSource copy = new IngestionSource.Builder(original).build(); + + assertEquals(original.isWarmupEnabled(), copy.isWarmupEnabled()); + assertEquals(original.getWarmupTimeout(), copy.getWarmupTimeout()); + assertEquals(original.getWarmupLagThreshold(), copy.getWarmupLagThreshold()); + assertEquals(original.isWarmupFailOnTimeout(), copy.isWarmupFailOnTimeout()); + } + + public void testWarmupConfigClass() { + IngestionSource.WarmupConfig config1 = new IngestionSource.WarmupConfig(true, TimeValue.timeValueMinutes(10), 100, true); + + assertEquals(true, config1.isEnabled()); + assertEquals(TimeValue.timeValueMinutes(10), config1.getTimeout()); + assertEquals(100, config1.getLagThreshold()); + assertEquals(true, config1.isFailOnTimeout()); + + // Test equals and hashCode + IngestionSource.WarmupConfig config2 = new IngestionSource.WarmupConfig(true, TimeValue.timeValueMinutes(10), 100, true); + assertEquals(config1, config2); + assertEquals(config1.hashCode(), config2.hashCode()); + + // Test inequality + IngestionSource.WarmupConfig config3 = new IngestionSource.WarmupConfig(false, TimeValue.timeValueMinutes(10), 100, true); + assertNotEquals(config1, config3); + + // Test toString + String expectedToString = "WarmupConfig{enabled=true, timeout=10m, lagThreshold=100, failOnTimeout=true}"; + assertEquals(expectedToString, config1.toString()); + } + + public void testSetWarmupConfig() { + IngestionSource.WarmupConfig warmupConfig = new IngestionSource.WarmupConfig(true, TimeValue.timeValueMinutes(15), 200, true); + + IngestionSource source = new IngestionSource.Builder("type").setWarmupConfig(warmupConfig).build(); + + assertEquals(warmupConfig, source.getWarmupConfig()); + assertTrue(source.isWarmupEnabled()); + assertEquals(TimeValue.timeValueMinutes(15), source.getWarmupTimeout()); + assertEquals(200, source.getWarmupLagThreshold()); + assertTrue(source.isWarmupFailOnTimeout()); + } } diff --git a/server/src/test/java/org/opensearch/indices/pollingingest/DefaultStreamPollerTests.java b/server/src/test/java/org/opensearch/indices/pollingingest/DefaultStreamPollerTests.java index ff36de1e525d3..f69682cfd2202 100644 --- a/server/src/test/java/org/opensearch/indices/pollingingest/DefaultStreamPollerTests.java +++ b/server/src/test/java/org/opensearch/indices/pollingingest/DefaultStreamPollerTests.java @@ -95,7 +95,11 @@ public void setUp() throws Exception { 1000, 10000, indexSettings, - new DefaultIngestionMessageMapper() + new DefaultIngestionMessageMapper(), + false, // warmupEnabled - disabled for existing tests + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout ); partitionedBlockingQueueContainer.startProcessorThreads(); } @@ -169,7 +173,11 @@ public void testResetStateEarliest() throws InterruptedException { 1000, 10000, indexSettings, - new DefaultIngestionMessageMapper() + new DefaultIngestionMessageMapper(), + false, // warmupEnabled + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout ); CountDownLatch latch = new CountDownLatch(2); doAnswer(invocation -> { @@ -203,7 +211,11 @@ public void testResetStateLatest() throws InterruptedException { 1000, 10000, indexSettings, - new DefaultIngestionMessageMapper() + new DefaultIngestionMessageMapper(), + false, // warmupEnabled + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout ); // Set up latch to wait for 2 messages to be processed @@ -245,7 +257,11 @@ public void testResetStateRewindByOffset() throws InterruptedException { 1000, 10000, indexSettings, - new DefaultIngestionMessageMapper() + new DefaultIngestionMessageMapper(), + false, // warmupEnabled + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout ); CountDownLatch latch = new CountDownLatch(1); doAnswer(invocation -> { @@ -331,7 +347,11 @@ public void testDropErrorIngestionStrategy() throws TimeoutException, Interrupte 1000, 10000, indexSettings, - new DefaultIngestionMessageMapper() + new DefaultIngestionMessageMapper(), + false, // warmupEnabled + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout ); poller.start(); Thread.sleep(sleepTime); @@ -394,7 +414,11 @@ public void testBlockErrorIngestionStrategy() throws TimeoutException, Interrupt 1000, 10000, indexSettings, - new DefaultIngestionMessageMapper() + new DefaultIngestionMessageMapper(), + false, // warmupEnabled + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout ); poller.start(); Thread.sleep(sleepTime); @@ -431,7 +455,11 @@ public void testProcessingErrorWithBlockErrorIngestionStrategy() throws TimeoutE 1000, 10000, indexSettings, - new DefaultIngestionMessageMapper() + new DefaultIngestionMessageMapper(), + false, // warmupEnabled + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout ); poller.start(); Thread.sleep(sleepTime); @@ -504,7 +532,11 @@ public void testPersistedBatchStartPointer() throws TimeoutException, Interrupte 1000, 10000, indexSettings, - new DefaultIngestionMessageMapper() + new DefaultIngestionMessageMapper(), + false, // warmupEnabled + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout ); poller.start(); Thread.sleep(sleepTime); @@ -574,7 +606,11 @@ public void testConsumerInitializationRetry() throws Exception { 1000, 10000, indexSettings, - new DefaultIngestionMessageMapper() + new DefaultIngestionMessageMapper(), + false, // warmupEnabled + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout ); poller.start(); @@ -617,7 +653,11 @@ public void testConsumerReinitializationAfterProcessingMessages() throws Excepti 1000, 10000, indexSettings, - new DefaultIngestionMessageMapper() + new DefaultIngestionMessageMapper(), + false, // warmupEnabled + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout ); // Start and wait for 2 messages to be processed @@ -673,7 +713,11 @@ public void testConsumerReinitializationWithNoInitialMessages() throws Exception 1000, 10000, indexSettings, - new DefaultIngestionMessageMapper() + new DefaultIngestionMessageMapper(), + false, // warmupEnabled + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout ); // Start poller @@ -717,10 +761,327 @@ public void testGetBatchStartPointerWithNullInitialPointer() { 1000, 10000, indexSettings, - new DefaultIngestionMessageMapper() + new DefaultIngestionMessageMapper(), + false, // warmupEnabled + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout ); // When all queues return null and initialBatchStartPointer is null, getBatchStartPointer should return null assertNull(poller.getBatchStartPointer()); } + + // ==================== Warmup Tests ==================== + + public void testWarmupDisabledBehavior() { + // When warmup is disabled, isWarmupComplete should return true immediately + // The default poller in setUp has warmupEnabled=false + assertTrue(poller.isWarmupComplete()); // Warmup disabled means it's considered complete + + // Create another poller with warmup explicitly disabled + DefaultStreamPoller warmupDisabledPoller = new DefaultStreamPoller( + new FakeIngestionSource.FakeIngestionShardPointer(0), + fakeConsumerFactory, + "", + 0, + partitionedBlockingQueueContainer, + StreamPoller.ResetState.NONE, + "", + errorStrategy, + StreamPoller.State.NONE, + 1000, + 1000, + 10000, + indexSettings, + new DefaultIngestionMessageMapper(), + false, // warmupEnabled = false + 300000, + 0, + false + ); + + // Warmup should be considered complete when disabled + assertTrue(warmupDisabledPoller.isWarmupComplete()); + warmupDisabledPoller.close(); + } + + public void testWarmupCompletesImmediatelyWhenLagIsZero() throws InterruptedException { + // Create a poller with warmup enabled and lag threshold of 0 + // FakeIngestionSource returns 0 for lag by default + DefaultStreamPoller warmupPoller = new DefaultStreamPoller( + new FakeIngestionSource.FakeIngestionShardPointer(0), + fakeConsumerFactory, + "", + 0, + partitionedBlockingQueueContainer, + StreamPoller.ResetState.NONE, + "", + errorStrategy, + StreamPoller.State.NONE, + 1000, + 1000, + 10000, + indexSettings, + new DefaultIngestionMessageMapper(), + true, // warmupEnabled = true + 300000, // warmupTimeoutMs = 5 minutes + 0, // warmupLagThreshold = 0 (fully caught up) + false // warmupFailOnTimeout + ); + + // Initially warmup is not complete + assertFalse(warmupPoller.isWarmupComplete()); + + // Start the poller - it should complete warmup quickly since lag is 0 + warmupPoller.start(); + + // Wait for warmup with timeout + boolean completed = warmupPoller.awaitWarmupComplete(5000); + assertTrue("Warmup should complete when lag is at threshold", completed); + assertTrue(warmupPoller.isWarmupComplete()); + + warmupPoller.close(); + } + + public void testWarmupTimesOutAndProceeds() throws InterruptedException, TimeoutException { + // Create a mock consumer factory that always reports high lag + IngestionConsumerFactory mockFactory = mock(IngestionConsumerFactory.class); + IngestionShardConsumer mockConsumer = mock(IngestionShardConsumer.class); + when(mockFactory.createShardConsumer(anyString(), anyInt())).thenReturn(mockConsumer); + when(mockConsumer.getPointerBasedLag(any())).thenReturn(1000L); // High lag + when(mockConsumer.readNext(anyLong(), anyInt())).thenReturn(Collections.emptyList()); + + DefaultStreamPoller warmupPoller = new DefaultStreamPoller( + new FakeIngestionSource.FakeIngestionShardPointer(0), + mockFactory, + "", + 0, + partitionedBlockingQueueContainer, + StreamPoller.ResetState.NONE, + "", + errorStrategy, + StreamPoller.State.NONE, + 1000, + 1000, + 10000, + indexSettings, + new DefaultIngestionMessageMapper(), + true, // warmupEnabled = true + 500, // warmupTimeoutMs = 500ms (short timeout for test) + 0, // warmupLagThreshold = 0 + false // warmupFailOnTimeout = false (proceed on timeout) + ); + + warmupPoller.start(); + + // Wait for timeout - should eventually complete due to timeout + boolean completed = warmupPoller.awaitWarmupComplete(2000); + // Even if lag never reaches threshold, warmup completes on timeout + assertTrue("Warmup should complete after timeout", warmupPoller.isWarmupComplete()); + + warmupPoller.close(); + } + + public void testWarmupStateTransitions() throws InterruptedException { + DefaultStreamPoller warmupPoller = new DefaultStreamPoller( + new FakeIngestionSource.FakeIngestionShardPointer(0), + fakeConsumerFactory, + "", + 0, + partitionedBlockingQueueContainer, + StreamPoller.ResetState.NONE, + "", + errorStrategy, + StreamPoller.State.NONE, + 1000, + 1000, + 10000, + indexSettings, + new DefaultIngestionMessageMapper(), + true, // warmupEnabled = true + 300000, // warmupTimeoutMs + 0, // warmupLagThreshold + false // warmupFailOnTimeout + ); + + // Initial state should be NONE + assertEquals(StreamPoller.State.NONE, warmupPoller.getState()); + + warmupPoller.start(); + + // Give it time to start + Thread.sleep(200); + + // State should eventually transition from WARMING_UP to POLLING/PROCESSING + // (since lag should be 0 from fake source) + warmupPoller.awaitWarmupComplete(5000); + assertTrue(warmupPoller.isWarmupComplete()); + + warmupPoller.close(); + } + + public void testIsWarmupFailOnTimeout() { + // Test with fail on timeout = true + DefaultStreamPoller failOnTimeoutPoller = new DefaultStreamPoller( + new FakeIngestionSource.FakeIngestionShardPointer(0), + fakeConsumerFactory, + "", + 0, + partitionedBlockingQueueContainer, + StreamPoller.ResetState.NONE, + "", + errorStrategy, + StreamPoller.State.NONE, + 1000, + 1000, + 10000, + indexSettings, + new DefaultIngestionMessageMapper(), + true, // warmupEnabled + 300000, + 0, + true // warmupFailOnTimeout = true + ); + + assertTrue(failOnTimeoutPoller.isWarmupFailOnTimeout()); + failOnTimeoutPoller.close(); + + // Test with fail on timeout = false + DefaultStreamPoller proceedOnTimeoutPoller = new DefaultStreamPoller( + new FakeIngestionSource.FakeIngestionShardPointer(0), + fakeConsumerFactory, + "", + 0, + partitionedBlockingQueueContainer, + StreamPoller.ResetState.NONE, + "", + errorStrategy, + StreamPoller.State.NONE, + 1000, + 1000, + 10000, + indexSettings, + new DefaultIngestionMessageMapper(), + true, // warmupEnabled + 300000, + 0, + false // warmupFailOnTimeout = false + ); + + assertFalse(proceedOnTimeoutPoller.isWarmupFailOnTimeout()); + proceedOnTimeoutPoller.close(); + } + + public void testWarmupCompletesViaTimeoutWhenPointerLagNegative() throws InterruptedException, TimeoutException { + // Create a mock consumer factory that returns negative pointer-based lag (unsupported) + IngestionConsumerFactory mockFactory = mock(IngestionConsumerFactory.class); + IngestionShardConsumer mockConsumer = mock(IngestionShardConsumer.class); + when(mockFactory.createShardConsumer(anyString(), anyInt())).thenReturn(mockConsumer); + when(mockConsumer.getPointerBasedLag(any())).thenReturn(-1L); // Negative means unsupported + when(mockConsumer.readNext(anyLong(), anyInt())).thenReturn(Collections.emptyList()); + + DefaultStreamPoller warmupPoller = new DefaultStreamPoller( + new FakeIngestionSource.FakeIngestionShardPointer(0), + mockFactory, + "", + 0, + partitionedBlockingQueueContainer, + StreamPoller.ResetState.NONE, + "", + errorStrategy, + StreamPoller.State.NONE, + 1000, + 1000, + 10000, + indexSettings, + new DefaultIngestionMessageMapper(), + true, // warmupEnabled = true + 500, // warmupTimeoutMs = 500ms (short for test) + 0, // warmupLagThreshold = 0 + false // warmupFailOnTimeout + ); + + warmupPoller.start(); + + // Should eventually complete via timeout since negative lag won't satisfy threshold + boolean completed = warmupPoller.awaitWarmupComplete(2000); + assertTrue("Warmup should complete after timeout when pointer lag is unsupported", warmupPoller.isWarmupComplete()); + + warmupPoller.close(); + } + + public void testWarmupWithNonZeroLagThreshold() throws InterruptedException, TimeoutException { + // Create a mock consumer factory that reports lag slightly above threshold then drops to threshold + IngestionConsumerFactory mockFactory = mock(IngestionConsumerFactory.class); + IngestionShardConsumer mockConsumer = mock(IngestionShardConsumer.class); + when(mockFactory.createShardConsumer(anyString(), anyInt())).thenReturn(mockConsumer); + // Return lag of 50 which is below threshold of 100 + when(mockConsumer.getPointerBasedLag(any())).thenReturn(50L); + when(mockConsumer.readNext(anyLong(), anyInt())).thenReturn(Collections.emptyList()); + + DefaultStreamPoller warmupPoller = new DefaultStreamPoller( + new FakeIngestionSource.FakeIngestionShardPointer(0), + mockFactory, + "", + 0, + partitionedBlockingQueueContainer, + StreamPoller.ResetState.NONE, + "", + errorStrategy, + StreamPoller.State.NONE, + 1000, + 1000, + 10000, + indexSettings, + new DefaultIngestionMessageMapper(), + true, // warmupEnabled = true + 30000, // warmupTimeoutMs = 30s + 100, // warmupLagThreshold = 100 (lag of 50 should be below this) + false // warmupFailOnTimeout + ); + + warmupPoller.start(); + + // Should complete quickly since lag (50) is below threshold (100) + boolean completed = warmupPoller.awaitWarmupComplete(5000); + assertTrue("Warmup should complete when lag is below threshold", completed); + assertTrue(warmupPoller.isWarmupComplete()); + + warmupPoller.close(); + } + + public void testWarmupAwaitReturnsImmediatelyWhenAlreadyComplete() throws InterruptedException { + // Create a poller with warmup disabled (which means warmup is considered complete) + DefaultStreamPoller warmupPoller = new DefaultStreamPoller( + new FakeIngestionSource.FakeIngestionShardPointer(0), + fakeConsumerFactory, + "", + 0, + partitionedBlockingQueueContainer, + StreamPoller.ResetState.NONE, + "", + errorStrategy, + StreamPoller.State.NONE, + 1000, + 1000, + 10000, + indexSettings, + new DefaultIngestionMessageMapper(), + false, // warmupEnabled = false + 300000, + 0, + false + ); + + // Should return immediately without blocking since warmup is disabled + long startTime = System.currentTimeMillis(); + boolean completed = warmupPoller.awaitWarmupComplete(60000); + long elapsed = System.currentTimeMillis() - startTime; + + assertTrue("Should return true when warmup is disabled", completed); + assertTrue("Should return immediately (< 1000ms)", elapsed < 1000); + + warmupPoller.close(); + } } From a03dc8b9efb60402e8625abaa25af467cd56faa0 Mon Sep 17 00:00:00 2001 From: Kaustubh Butte Date: Thu, 12 Feb 2026 12:50:57 +0530 Subject: [PATCH 02/12] Refactor warmup to use WarmupConfig and change default lag_threshold to 100 - Change default lag_threshold from 0 to 100 messages per msfroh's suggestion - Add WarmupConfig.DEFAULT constant for cleaner initialization - Refactor DefaultStreamPoller to accept WarmupConfig object instead of 4 individual parameters - Extract warmup handling in IndexShard to handlePullBasedIngestionWarmup() method - Update tests to use WarmupConfig Signed-off-by: Kaustubh Butte --- .../cluster/metadata/IndexMetadata.java | 2 +- .../cluster/metadata/IngestionSource.java | 5 + .../index/engine/IngestionEngine.java | 5 +- .../opensearch/index/shard/IndexShard.java | 29 +++-- .../pollingingest/DefaultStreamPoller.java | 83 ++++---------- .../DefaultStreamPollerTests.java | 106 ++++-------------- 6 files changed, 69 insertions(+), 161 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 6517a44c9d936..55fd611cac911 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -1061,7 +1061,7 @@ public Iterator> settings() { public static final String SETTING_INGESTION_SOURCE_WARMUP_LAG_THRESHOLD = "index.ingestion_source.warmup.lag_threshold"; public static final Setting INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING = Setting.longSetting( SETTING_INGESTION_SOURCE_WARMUP_LAG_THRESHOLD, - 0L, + 100L, 0L, Property.IndexScope, Property.Final diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java index ff11df9d3ca1d..5d4391fb14935 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java @@ -272,6 +272,11 @@ public String toString() { */ @ExperimentalApi public static class WarmupConfig { + /** + * Default warmup configuration matching IndexMetadata setting defaults. + */ + public static final WarmupConfig DEFAULT = new WarmupConfig(true, TimeValue.timeValueMinutes(5), 100L, false); + private final boolean enabled; private final TimeValue timeout; private final long lagThreshold; diff --git a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java index ffeabae2fd119..f839ffe79f797 100644 --- a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java @@ -155,10 +155,7 @@ private void initializeStreamPoller( .pointerBasedLagUpdateInterval(ingestionSource.getPointerBasedLagUpdateInterval().millis()) .mapperType(ingestionSource.getMapperType()) .mapperSettings(ingestionSource.getMapperSettings()) - .warmupEnabled(ingestionSource.isWarmupEnabled()) - .warmupTimeoutMs(ingestionSource.getWarmupTimeout().millis()) - .warmupLagThreshold(ingestionSource.getWarmupLagThreshold()) - .warmupFailOnTimeout(ingestionSource.isWarmupFailOnTimeout()) + .warmupConfig(ingestionSource.getWarmupConfig()) .build(); registerStreamPollerListener(); diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 2e58b7224bede..ae8a50fda5c93 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -2490,15 +2490,7 @@ public void postRecovery(String reason) throws IndexShardStartedException, Index // Wait for ingestion warmup if enabled (pull-based ingestion only) Engine engine = getEngine(); - if (engine instanceof IngestionEngine) { - IngestionEngine ingestionEngine = (IngestionEngine) engine; - try { - ingestionEngine.awaitWarmupComplete(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new OpenSearchException("Interrupted waiting for ingestion warmup", e); - } - } + handlePullBasedIngestionWarmup(engine); synchronized (mutex) { if (state == IndexShardState.CLOSED) { @@ -2513,6 +2505,25 @@ public void postRecovery(String reason) throws IndexShardStartedException, Index } } + /** + * Handles warmup for pull-based ingestion (PBI) engines. + * When warmup is enabled, this method blocks until the shard has caught up with the streaming source + * or until the configured timeout is reached. + * + * @param engine the engine to check for warmup + */ + private void handlePullBasedIngestionWarmup(Engine engine) { + if (engine instanceof IngestionEngine) { + IngestionEngine ingestionEngine = (IngestionEngine) engine; + try { + ingestionEngine.awaitWarmupComplete(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new OpenSearchException("Interrupted waiting for ingestion warmup", e); + } + } + } + /** * called before starting to copy index files over */ diff --git a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java index 8c8ea6cd9fd7d..06a2e2eebd5c9 100644 --- a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java +++ b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java @@ -62,10 +62,7 @@ public class DefaultStreamPoller implements StreamPoller { private volatile long lastPointerBasedLagUpdateTime = 0; // Warmup configuration and state - private final boolean warmupEnabled; - private final long warmupTimeoutMs; - private final long warmupLagThreshold; - private final boolean warmupFailOnTimeout; + private final IngestionSource.WarmupConfig warmupConfig; private volatile boolean warmupComplete = false; private volatile long warmupStartTime = 0; private final CountDownLatch warmupLatch = new CountDownLatch(1); @@ -119,10 +116,7 @@ private DefaultStreamPoller( long pointerBasedLagUpdateIntervalMs, IngestionMessageMapper.MapperType mapperType, Map mapperSettings, - boolean warmupEnabled, - long warmupTimeoutMs, - long warmupLagThreshold, - boolean warmupFailOnTimeout + IngestionSource.WarmupConfig warmupConfig ) { this( startPointer, @@ -139,10 +133,8 @@ private DefaultStreamPoller( pointerBasedLagUpdateIntervalMs, ingestionEngine.config().getIndexSettings(), IngestionMessageMapper.create(mapperType.getName(), shardId, mapperSettings), - warmupEnabled, - warmupTimeoutMs, - warmupLagThreshold, - warmupFailOnTimeout + IngestionMessageMapper.create(mapperType.getName(), shardId), + warmupConfig ); } @@ -164,10 +156,7 @@ private DefaultStreamPoller( long pointerBasedLagUpdateIntervalMs, IndexSettings indexSettings, IngestionMessageMapper messageMapper, - boolean warmupEnabled, - long warmupTimeoutMs, - long warmupLagThreshold, - boolean warmupFailOnTimeout + IngestionSource.WarmupConfig warmupConfig ) { this.consumerFactory = Objects.requireNonNull(consumerFactory); this.consumerClientId = Objects.requireNonNull(consumerClientId); @@ -186,15 +175,12 @@ private DefaultStreamPoller( this.errorStrategy = errorStrategy; this.indexName = indexSettings.getIndex().getName(); this.messageMapper = Objects.requireNonNull(messageMapper); - this.warmupEnabled = warmupEnabled; - this.warmupTimeoutMs = warmupTimeoutMs; - this.warmupLagThreshold = warmupLagThreshold; - this.warmupFailOnTimeout = warmupFailOnTimeout; + this.warmupConfig = Objects.requireNonNull(warmupConfig); // handle initial poller states this.paused = initialState == State.PAUSED; // If warmup is disabled, mark as complete immediately - if (!warmupEnabled) { + if (!warmupConfig.isEnabled()) { this.warmupComplete = true; } } @@ -227,7 +213,7 @@ protected void startPoll() { logger.info("Starting poller for shard {}", shardId); // Initialize warmup if enabled - if (warmupEnabled && !warmupComplete) { + if (warmupConfig.isEnabled() && !warmupComplete) { warmupStartTime = System.currentTimeMillis(); state = State.WARMING_UP; logger.info("Starting warmup phase for index {} shard {}, waiting for lag to catch up", indexName, shardId); @@ -251,7 +237,7 @@ protected void startPoll() { updatePointerBasedLagIfNeeded(); // Check warmup status if not yet complete - if (!warmupComplete && warmupEnabled) { + if (!warmupComplete && warmupConfig.isEnabled()) { checkWarmupStatus(); } @@ -400,19 +386,19 @@ public boolean isClosed() { @Override public boolean isWarmupComplete() { - return warmupComplete || !warmupEnabled; + return warmupComplete || !warmupConfig.isEnabled(); } /** * Returns true if shard initialization should fail when warmup times out. */ public boolean isWarmupFailOnTimeout() { - return warmupFailOnTimeout; + return warmupConfig.isFailOnTimeout(); } @Override public boolean awaitWarmupComplete(long timeoutMs) throws InterruptedException { - if (!warmupEnabled || isWarmupComplete()) { + if (!warmupConfig.isEnabled() || isWarmupComplete()) { return true; } @@ -439,11 +425,11 @@ public boolean awaitWarmupComplete(long timeoutMs) throws InterruptedException { */ private void checkWarmupStatus() { long currentLag = cachedPointerBasedLag; - long threshold = warmupLagThreshold; + long threshold = warmupConfig.getLagThreshold(); long elapsedTime = System.currentTimeMillis() - warmupStartTime; boolean lagBelowThreshold = currentLag >= 0 && currentLag <= threshold; - boolean timeoutReached = elapsedTime >= warmupTimeoutMs; + boolean timeoutReached = elapsedTime >= warmupConfig.getTimeout().millis(); if (lagBelowThreshold) { warmupComplete = true; @@ -722,11 +708,8 @@ public static class Builder { private long pointerBasedLagUpdateIntervalMs = 10000; private IngestionMessageMapper.MapperType mapperType = IngestionMessageMapper.MapperType.DEFAULT; private Map mapperSettings = Collections.emptyMap(); - // Warmup configuration - defaults match IndexMetadata settings - private boolean warmupEnabled = true; - private long warmupTimeoutMs = 300000; // 5 minutes - private long warmupLagThreshold = 0; - private boolean warmupFailOnTimeout = false; + // Warmup configuration - default matches IndexMetadata settings + private IngestionSource.WarmupConfig warmupConfig = IngestionSource.WarmupConfig.DEFAULT; /** * Initialize the builder with mandatory parameters @@ -836,33 +819,10 @@ public Builder mapperSettings(Map mapperSettings) { /** * Set warmup enabled + * Set warmup configuration */ - public Builder warmupEnabled(boolean warmupEnabled) { - this.warmupEnabled = warmupEnabled; - return this; - } - - /** - * Set warmup timeout in milliseconds - */ - public Builder warmupTimeoutMs(long warmupTimeoutMs) { - this.warmupTimeoutMs = warmupTimeoutMs; - return this; - } - - /** - * Set warmup lag threshold (pointer-based) - */ - public Builder warmupLagThreshold(long warmupLagThreshold) { - this.warmupLagThreshold = warmupLagThreshold; - return this; - } - - /** - * Set whether shard initialization should fail on warmup timeout - */ - public Builder warmupFailOnTimeout(boolean warmupFailOnTimeout) { - this.warmupFailOnTimeout = warmupFailOnTimeout; + public Builder warmupConfig(IngestionSource.WarmupConfig warmupConfig) { + this.warmupConfig = Objects.requireNonNull(warmupConfig); return this; } @@ -887,10 +847,7 @@ public DefaultStreamPoller build() { pointerBasedLagUpdateIntervalMs, mapperType, mapperSettings, - warmupEnabled, - warmupTimeoutMs, - warmupLagThreshold, - warmupFailOnTimeout + warmupConfig ); } } diff --git a/server/src/test/java/org/opensearch/indices/pollingingest/DefaultStreamPollerTests.java b/server/src/test/java/org/opensearch/indices/pollingingest/DefaultStreamPollerTests.java index f69682cfd2202..fa0d2e09a7fe9 100644 --- a/server/src/test/java/org/opensearch/indices/pollingingest/DefaultStreamPollerTests.java +++ b/server/src/test/java/org/opensearch/indices/pollingingest/DefaultStreamPollerTests.java @@ -16,6 +16,7 @@ import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.metadata.IngestionSource; import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; import org.opensearch.core.rest.RestStatus; import org.opensearch.index.IndexSettings; import org.opensearch.index.IngestionConsumerFactory; @@ -96,10 +97,7 @@ public void setUp() throws Exception { 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled - disabled for existing tests - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); partitionedBlockingQueueContainer.startProcessorThreads(); } @@ -174,10 +172,7 @@ public void testResetStateEarliest() throws InterruptedException { 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); CountDownLatch latch = new CountDownLatch(2); doAnswer(invocation -> { @@ -212,10 +207,7 @@ public void testResetStateLatest() throws InterruptedException { 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); // Set up latch to wait for 2 messages to be processed @@ -258,10 +250,7 @@ public void testResetStateRewindByOffset() throws InterruptedException { 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); CountDownLatch latch = new CountDownLatch(1); doAnswer(invocation -> { @@ -348,10 +337,7 @@ public void testDropErrorIngestionStrategy() throws TimeoutException, Interrupte 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); poller.start(); Thread.sleep(sleepTime); @@ -415,10 +401,7 @@ public void testBlockErrorIngestionStrategy() throws TimeoutException, Interrupt 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); poller.start(); Thread.sleep(sleepTime); @@ -456,10 +439,7 @@ public void testProcessingErrorWithBlockErrorIngestionStrategy() throws TimeoutE 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); poller.start(); Thread.sleep(sleepTime); @@ -533,10 +513,7 @@ public void testPersistedBatchStartPointer() throws TimeoutException, Interrupte 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); poller.start(); Thread.sleep(sleepTime); @@ -607,10 +584,7 @@ public void testConsumerInitializationRetry() throws Exception { 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); poller.start(); @@ -654,10 +628,7 @@ public void testConsumerReinitializationAfterProcessingMessages() throws Excepti 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); // Start and wait for 2 messages to be processed @@ -714,10 +685,7 @@ public void testConsumerReinitializationWithNoInitialMessages() throws Exception 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); // Start poller @@ -762,10 +730,7 @@ public void testGetBatchStartPointerWithNullInitialPointer() { 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); // When all queues return null and initialBatchStartPointer is null, getBatchStartPointer should return null @@ -795,10 +760,7 @@ public void testWarmupDisabledBehavior() { 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled = false - 300000, - 0, - false + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); // Warmup should be considered complete when disabled @@ -824,10 +786,7 @@ public void testWarmupCompletesImmediatelyWhenLagIsZero() throws InterruptedExce 10000, indexSettings, new DefaultIngestionMessageMapper(), - true, // warmupEnabled = true - 300000, // warmupTimeoutMs = 5 minutes - 0, // warmupLagThreshold = 0 (fully caught up) - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(300000), 0, false) ); // Initially warmup is not complete @@ -867,10 +826,7 @@ public void testWarmupTimesOutAndProceeds() throws InterruptedException, Timeout 10000, indexSettings, new DefaultIngestionMessageMapper(), - true, // warmupEnabled = true - 500, // warmupTimeoutMs = 500ms (short timeout for test) - 0, // warmupLagThreshold = 0 - false // warmupFailOnTimeout = false (proceed on timeout) + new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(500), 0, false) ); warmupPoller.start(); @@ -899,10 +855,7 @@ public void testWarmupStateTransitions() throws InterruptedException { 10000, indexSettings, new DefaultIngestionMessageMapper(), - true, // warmupEnabled = true - 300000, // warmupTimeoutMs - 0, // warmupLagThreshold - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(300000), 0, false) ); // Initial state should be NONE @@ -938,10 +891,7 @@ public void testIsWarmupFailOnTimeout() { 10000, indexSettings, new DefaultIngestionMessageMapper(), - true, // warmupEnabled - 300000, - 0, - true // warmupFailOnTimeout = true + new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(300000), 0, true) ); assertTrue(failOnTimeoutPoller.isWarmupFailOnTimeout()); @@ -963,10 +913,7 @@ public void testIsWarmupFailOnTimeout() { 10000, indexSettings, new DefaultIngestionMessageMapper(), - true, // warmupEnabled - 300000, - 0, - false // warmupFailOnTimeout = false + new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(300000), 0, false) ); assertFalse(proceedOnTimeoutPoller.isWarmupFailOnTimeout()); @@ -996,10 +943,7 @@ public void testWarmupCompletesViaTimeoutWhenPointerLagNegative() throws Interru 10000, indexSettings, new DefaultIngestionMessageMapper(), - true, // warmupEnabled = true - 500, // warmupTimeoutMs = 500ms (short for test) - 0, // warmupLagThreshold = 0 - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(500), 0, false) ); warmupPoller.start(); @@ -1035,10 +979,7 @@ public void testWarmupWithNonZeroLagThreshold() throws InterruptedException, Tim 10000, indexSettings, new DefaultIngestionMessageMapper(), - true, // warmupEnabled = true - 30000, // warmupTimeoutMs = 30s - 100, // warmupLagThreshold = 100 (lag of 50 should be below this) - false // warmupFailOnTimeout + new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(30000), 100, false) ); warmupPoller.start(); @@ -1068,10 +1009,7 @@ public void testWarmupAwaitReturnsImmediatelyWhenAlreadyComplete() throws Interr 10000, indexSettings, new DefaultIngestionMessageMapper(), - false, // warmupEnabled = false - 300000, - 0, - false + new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) ); // Should return immediately without blocking since warmup is disabled From 0d452c6227700391dd04ea0023a3bb7270690a3c Mon Sep 17 00:00:00 2001 From: Kaustubh Butte Date: Thu, 12 Feb 2026 13:00:19 +0530 Subject: [PATCH 03/12] Skip warmup when poller is paused - Add isPaused() check in IngestionEngine.awaitWarmupComplete() to skip waiting - Add paused check in DefaultStreamPoller.checkWarmupStatus() to mark warmup complete Signed-off-by: Kaustubh Butte --- .../java/org/opensearch/index/engine/IngestionEngine.java | 2 +- .../indices/pollingingest/DefaultStreamPoller.java | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java index f839ffe79f797..a667caa476fd8 100644 --- a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java @@ -680,7 +680,7 @@ public boolean isWarmupComplete() { */ public void awaitWarmupComplete() throws InterruptedException { IngestionSource ingestionSource = engineConfig.getIndexSettings().getIndexMetadata().getIngestionSource(); - if (ingestionSource == null || !ingestionSource.isWarmupEnabled()) { + if (ingestionSource == null || !ingestionSource.isWarmupEnabled() || streamPoller.isPaused()) { return; } diff --git a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java index 06a2e2eebd5c9..831ca1abbb0a2 100644 --- a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java +++ b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java @@ -424,6 +424,14 @@ public boolean awaitWarmupComplete(long timeoutMs) throws InterruptedException { * is called. */ private void checkWarmupStatus() { + // Skip warmup if poller is paused + if (paused) { + warmupComplete = true; + warmupLatch.countDown(); + logger.info("Warmup skipped for index {} shard {} - poller is paused", indexName, shardId); + return; + } + long currentLag = cachedPointerBasedLag; long threshold = warmupConfig.getLagThreshold(); From 1f4b99b0336fbff3078a15abbd634572c665a6ad Mon Sep 17 00:00:00 2001 From: Kaustubh Butte Date: Thu, 12 Feb 2026 13:04:04 +0530 Subject: [PATCH 04/12] Fix CHANGELOG conflicts with main Signed-off-by: Kaustubh Butte --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77c327b2bbba1..c001a2925d7fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add getWrappedScorer method to ProfileScorer for plugin access to wrapped scorers ([#20548](https://github.com/opensearch-project/OpenSearch/issues/20548)) - Support expected cluster name with validation in CCS Sniff mode ([#20532](https://github.com/opensearch-project/OpenSearch/pull/20532)) - Add security policy to allow `accessUnixDomainSocket` in `transport-grpc` module ([#20463](https://github.com/opensearch-project/OpenSearch/pull/20463)) +- Add range validations in query builder and field mapper ([#20497](https://github.com/opensearch-project/OpenSearch/issues/20497)) - [Workload Management] Enhance Scroll API support for autotagging ([#20151](https://github.com/opensearch-project/OpenSearch/pull/20151)) +- Add indices to search request slowlog ([#20588](https://github.com/opensearch-project/OpenSearch/pull/20588)) - Add warmup phase to wait for lag to catch up in pull-based ingestion before serving ([#20526](https://github.com/opensearch-project/OpenSearch/pull/20526)) ### Changed @@ -65,6 +67,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix array_index_out_of_bounds_exception with wildcard and aggregations ([#20842](https://github.com/opensearch-project/OpenSearch/pull/20842)) - - Delegate getMin/getMax methods for ExitableTerms ([#20775](https://github.com/opensearch-project/OpenSearch/pull/20775)) - Fix CriteriaBasedCodec to work with delegate codec. ([20442](https://github.com/opensearch-project/OpenSearch/pull/20442)) +- Fix WLM workload group creation failing due to updated_at clock skew ([#20486](https://github.com/opensearch-project/OpenSearch/pull/20486)) ### Dependencies - Bump shadow-gradle-plugin from 8.3.9 to 9.3.1 ([#20569](https://github.com/opensearch-project/OpenSearch/pull/20569)) From d905a85ae0cffafc0f21772246a6aef1123e7952 Mon Sep 17 00:00:00 2001 From: Kaustubh Butte Date: Wed, 18 Feb 2026 21:39:41 +0530 Subject: [PATCH 05/12] Add setState with warmup awareness and fix CHANGELOG - Add setStateWithWarmupAwareness method to keep WARMING_UP state during warmup - Replace direct state assignments with setStateWithWarmupAwareness calls - Update CHANGELOG with latest from main Signed-off-by: Kaustubh Butte --- .../pollingingest/DefaultStreamPoller.java | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java index 831ca1abbb0a2..cdeb1a8ee2f18 100644 --- a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java +++ b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java @@ -222,7 +222,7 @@ protected void startPoll() { while (true) { try { if (closed) { - state = State.CLOSED; + setStateWithWarmupAwareness(State.CLOSED); closeConsumer(); break; } @@ -242,7 +242,7 @@ protected void startPoll() { } if (paused || isWriteBlockEnabled) { - state = State.PAUSED; + setStateWithWarmupAwareness(State.PAUSED); try { Thread.sleep(DEFAULT_POLLER_SLEEP_PERIOD_MS); } catch (Throwable e) { @@ -251,7 +251,7 @@ protected void startPoll() { continue; } - state = State.POLLING; + setStateWithWarmupAwareness(State.POLLING); List> results; // Force the consumer to start from forcedShardPointer if available @@ -269,7 +269,7 @@ protected void startPoll() { continue; } - state = State.PROCESSING; + setStateWithWarmupAwareness(State.PROCESSING); // processRecords returns failed shard pointers. Update forcedShardPointer to the failed pointer to retry on next iteration // in case of failures forcedShardPointer = processRecords(results); @@ -389,6 +389,29 @@ public boolean isWarmupComplete() { return warmupComplete || !warmupConfig.isEnabled(); } + /** + * Sets the poller state with warmup-aware logic. + * During warmup, POLLING and PROCESSING states are reported as WARMING_UP + * to allow monitoring via the ingestion state API. + * + * @param newState the desired state to set + */ + private void setStateWithWarmupAwareness(State newState) { + // CLOSED and PAUSED always take effect + if (newState == State.CLOSED || newState == State.PAUSED) { + this.state = newState; + return; + } + + // During warmup, stay in WARMING_UP instead of POLLING/PROCESSING + if (!isWarmupComplete()) { + this.state = State.WARMING_UP; + return; + } + + this.state = newState; + } + /** * Returns true if shard initialization should fail when warmup times out. */ From 4ba82258a7317f8ed8c4d6bcc82867c4893c9122 Mon Sep 17 00:00:00 2001 From: Kaustubh Butte Date: Wed, 18 Feb 2026 22:20:06 +0530 Subject: [PATCH 06/12] Add warmup integration test and fix test defaults - Add testWarmupPhase IT to verify warmup functionality end-to-end - Fix IngestionSourceTests to expect lagThreshold=100 (new default) Signed-off-by: Kaustubh Butte --- .../plugin/kafka/IngestFromKafkaIT.java | 47 +++++++++++++++++++ .../metadata/IngestionSourceTests.java | 4 +- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java index af863d6519180..34450e58a2b42 100644 --- a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java +++ b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java @@ -1350,5 +1350,52 @@ public void testKafkaIngestionWithFieldMappingMapper_VariousConfigurations() thr && "diana".equals(docs.get("jkl").get("name")) && !docs.get("jkl").containsKey("expired"); }); + + public void testWarmupPhase() throws Exception { + // Step 1: Publish 10 messages before creating the index + for (int i = 0; i < 10; i++) { + produceData(Integer.toString(i), "name" + i, "25"); + } + + // Step 2: Start cluster + internalCluster().startClusterManagerOnlyNode(); + final String nodeA = internalCluster().startDataOnlyNode(); + + // Step 3: Create index with warmup enabled, lag_threshold=0, and long timeout + createIndex( + indexName, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("ingestion_source.type", "kafka") + .put("ingestion_source.pointer.init.reset", "earliest") + .put("ingestion_source.param.topic", topicName) + .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) + .put("ingestion_source.warmup.enabled", true) + .put("ingestion_source.warmup.lag_threshold", 0) + .put("ingestion_source.warmup.timeout", "10m") + .put("ingestion_source.all_active", true) + .build(), + "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" + ); + + ensureGreen(indexName); + + // Step 4: Wait for poller to enter POLLING state (warmup complete) + waitForState(() -> { + GetIngestionStateResponse ingestionState = getIngestionState(indexName); + return ingestionState.getShardStates().length == 1 + && ingestionState.getShardStates()[0].getPollerState().equalsIgnoreCase("polling"); + }); + + // Step 5: Validate all 10 documents are searchable + waitForSearchableDocs(10, List.of(nodeA)); + + // Step 6: Verify stats + PollingIngestStats stats = client(nodeA).admin().indices().prepareStats(indexName).get().getIndex(indexName).getShards()[0] + .getPollingIngestStats(); + assertNotNull(stats); + assertEquals(10L, stats.getMessageProcessorStats().totalProcessedCount()); + assertEquals(10L, stats.getConsumerStats().totalPolledCount()); } } diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java index 80a66465cebfe..f6e8c096f2a88 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java @@ -108,7 +108,7 @@ public void testToString() { .setErrorStrategy(DROP) .build(); String expected = - "IngestionSource{type='type',pointer_init_reset='PointerInitReset{type='RESET_BY_OFFSET', value=1000}',error_strategy='DROP', params={key=value}, maxPollSize=1000, pollTimeout=1000, numProcessorThreads=1, blockingQueueSize=100, allActiveIngestion=false, pointerBasedLagUpdateInterval=10s, mapperType='DEFAULT', mapperSettings={}, warmupConfig=WarmupConfig{enabled=true, timeout=5m, lagThreshold=0, failOnTimeout=false}}"; + "IngestionSource{type='type',pointer_init_reset='PointerInitReset{type='RESET_BY_OFFSET', value=1000}',error_strategy='DROP', params={key=value}, maxPollSize=1000, pollTimeout=1000, numProcessorThreads=1, blockingQueueSize=100, allActiveIngestion=false, pointerBasedLagUpdateInterval=10s, mapperType='DEFAULT', mapperSettings={}, warmupConfig=WarmupConfig{enabled=true, timeout=5m, lagThreshold=100, failOnTimeout=false}}"; assertEquals(expected, source.toString()); } @@ -204,7 +204,7 @@ public void testWarmupConfigurationDefaults() { assertTrue("Warmup should be enabled by default", source.isWarmupEnabled()); assertEquals(TimeValue.timeValueMinutes(5), source.getWarmupTimeout()); - assertEquals(0, source.getWarmupLagThreshold()); + assertEquals(100, source.getWarmupLagThreshold()); assertFalse("Should not fail on timeout by default", source.isWarmupFailOnTimeout()); } From 3b0184f42afa3066c73f768e7512eeab859fc348 Mon Sep 17 00:00:00 2001 From: Kaustubh Butte Date: Wed, 11 Mar 2026 11:15:34 +0530 Subject: [PATCH 07/12] Address review comments from msfroh and varunbharadwaj - Remove WarmupConfig.DEFAULT constant to avoid defaults drifting from IndexMetadata settings (msfroh) - Replace waitForSearchableDocs with direct refresh + assert in warmup IT to catch warmup issues immediately (varunbharadwaj) - Rename checkWarmupStatus() to updateWarmupStatus() for clarity (varunbharadwaj) Signed-off-by: Kaustubh Butte --- .../opensearch/plugin/kafka/IngestFromKafkaIT.java | 9 +++++++-- .../cluster/metadata/IngestionSource.java | 5 ----- .../indices/pollingingest/DefaultStreamPoller.java | 13 +++++++++---- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java index 34450e58a2b42..d5ae03f69f7f8 100644 --- a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java +++ b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java @@ -1350,6 +1350,7 @@ public void testKafkaIngestionWithFieldMappingMapper_VariousConfigurations() thr && "diana".equals(docs.get("jkl").get("name")) && !docs.get("jkl").containsKey("expired"); }); + } public void testWarmupPhase() throws Exception { // Step 1: Publish 10 messages before creating the index @@ -1388,8 +1389,12 @@ public void testWarmupPhase() throws Exception { && ingestionState.getShardStates()[0].getPollerState().equalsIgnoreCase("polling"); }); - // Step 5: Validate all 10 documents are searchable - waitForSearchableDocs(10, List.of(nodeA)); + // Step 5: Validate all 10 documents are searchable after warmup + // Refresh to make all docs visible, then assert directly (not waitForSearchableDocs) + // since all 10 docs should already be indexed during the warmup phase + client(nodeA).admin().indices().prepareRefresh(indexName).get(); + long docCount = client(nodeA).prepareSearch(indexName).setSize(0).get().getHits().getTotalHits().value(); + assertEquals("All 10 documents should be searchable immediately after warmup completes", 10L, docCount); // Step 6: Verify stats PollingIngestStats stats = client(nodeA).admin().indices().prepareStats(indexName).get().getIndex(indexName).getShards()[0] diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java index 5d4391fb14935..ff11df9d3ca1d 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java @@ -272,11 +272,6 @@ public String toString() { */ @ExperimentalApi public static class WarmupConfig { - /** - * Default warmup configuration matching IndexMetadata setting defaults. - */ - public static final WarmupConfig DEFAULT = new WarmupConfig(true, TimeValue.timeValueMinutes(5), 100L, false); - private final boolean enabled; private final TimeValue timeout; private final long lagThreshold; diff --git a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java index cdeb1a8ee2f18..aed88186e1f54 100644 --- a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java +++ b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java @@ -16,6 +16,7 @@ import org.opensearch.cluster.metadata.IngestionSource; import org.opensearch.common.Nullable; import org.opensearch.common.metrics.CounterMetric; +import org.opensearch.common.unit.TimeValue; import org.opensearch.index.IndexSettings; import org.opensearch.index.IngestionConsumerFactory; import org.opensearch.index.IngestionShardConsumer; @@ -133,7 +134,6 @@ private DefaultStreamPoller( pointerBasedLagUpdateIntervalMs, ingestionEngine.config().getIndexSettings(), IngestionMessageMapper.create(mapperType.getName(), shardId, mapperSettings), - IngestionMessageMapper.create(mapperType.getName(), shardId), warmupConfig ); } @@ -238,7 +238,7 @@ protected void startPoll() { // Check warmup status if not yet complete if (!warmupComplete && warmupConfig.isEnabled()) { - checkWarmupStatus(); + updateWarmupStatus(); } if (paused || isWriteBlockEnabled) { @@ -446,7 +446,7 @@ public boolean awaitWarmupComplete(long timeoutMs) throws InterruptedException { * Note: cachedPointerBasedLag is 0 by default and is only updated after updatePointerBasedLagIfNeeded() * is called. */ - private void checkWarmupStatus() { + private void updateWarmupStatus() { // Skip warmup if poller is paused if (paused) { warmupComplete = true; @@ -740,7 +740,12 @@ public static class Builder { private IngestionMessageMapper.MapperType mapperType = IngestionMessageMapper.MapperType.DEFAULT; private Map mapperSettings = Collections.emptyMap(); // Warmup configuration - default matches IndexMetadata settings - private IngestionSource.WarmupConfig warmupConfig = IngestionSource.WarmupConfig.DEFAULT; + private IngestionSource.WarmupConfig warmupConfig = new IngestionSource.WarmupConfig( + false, + TimeValue.timeValueMinutes(5), + 100L, + false + ); /** * Initialize the builder with mandatory parameters From 945036465f01d80c9269fc9723db3b554666fa75 Mon Sep 17 00:00:00 2001 From: Kaustubh Butte Date: Wed, 11 Mar 2026 15:02:02 +0530 Subject: [PATCH 08/12] Initialize cachedPointerBasedLag to -1 to prevent premature warmup completion Before lag is first fetched from the consumer, cachedPointerBasedLag was 0 which could satisfy lagBelowThreshold check (0 <= threshold) causing warmup to complete before real lag data is available. Using -1 as sentinel ensures the existing 'currentLag >= 0' guard prevents premature completion. Signed-off-by: Kaustubh Butte --- .../opensearch/indices/pollingingest/DefaultStreamPoller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java index aed88186e1f54..af0037fe3a22a 100644 --- a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java +++ b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java @@ -59,7 +59,7 @@ public class DefaultStreamPoller implements StreamPoller { private volatile boolean reinitializeConsumer; private volatile long lastPolledMessageTimestamp = 0; - private volatile long cachedPointerBasedLag = 0; + private volatile long cachedPointerBasedLag = -1; private volatile long lastPointerBasedLagUpdateTime = 0; // Warmup configuration and state From c0c81742bb127afa09bd64986fb670bc3d43257b Mon Sep 17 00:00:00 2001 From: Kaustubh Butte Date: Thu, 12 Mar 2026 00:19:45 +0530 Subject: [PATCH 09/12] Disable warmup in KafkaSingleNodeTests to fix test failure Tests that don't test warmup behavior need warmup.enabled=false since warmup (enabled by default) blocks shard initialization, causing createIndex assertAcked to fail with 'not all shards were started'. Signed-off-by: Kaustubh Butte --- CHANGELOG.md | 2 -- .../java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c001a2925d7fe..d73d368ebfdee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,8 +66,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix terms lookup subquery fetch limit reading from non-existent index setting instead of cluster `max_clause_count` ([#20823](https://github.com/opensearch-project/OpenSearch/pull/20823)) - Fix array_index_out_of_bounds_exception with wildcard and aggregations ([#20842](https://github.com/opensearch-project/OpenSearch/pull/20842)) - - Delegate getMin/getMax methods for ExitableTerms ([#20775](https://github.com/opensearch-project/OpenSearch/pull/20775)) -- Fix CriteriaBasedCodec to work with delegate codec. ([20442](https://github.com/opensearch-project/OpenSearch/pull/20442)) -- Fix WLM workload group creation failing due to updated_at clock skew ([#20486](https://github.com/opensearch-project/OpenSearch/pull/20486)) ### Dependencies - Bump shadow-gradle-plugin from 8.3.9 to 9.3.1 ([#20569](https://github.com/opensearch-project/OpenSearch/pull/20569)) diff --git a/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java b/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java index 428c7df1dde36..9a0990d7e1728 100644 --- a/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java +++ b/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java @@ -81,6 +81,7 @@ public void testPauseAndResumeAPIs() throws Exception { .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("index.replication.type", "SEGMENT") .put("ingestion_source.pointer_based_lag_update_interval", "0") + .put("ingestion_source.warmup.enabled", false) .build(), mappings ); @@ -175,6 +176,7 @@ public void testConsumerSettingUpdateWithMultipleProcessorThreads() throws Excep .put("ingestion_source.param.auto.offset.reset", "none") .put("ingestion_source.num_processor_threads", 5) .put("index.replication.type", "SEGMENT") + .put("ingestion_source.warmup.enabled", false) .build(), mappings ); From e8cf5c5b264fd9e85ae04f71b2b35b8ca79b2a25 Mon Sep 17 00:00:00 2001 From: Kaustubh Butte Date: Thu, 12 Mar 2026 09:49:49 +0530 Subject: [PATCH 10/12] Address review feedback: annotations, unused method, and comments - Change @ExperimentalApi to @PublicApi(since = "3.6.0") on WarmupConfig - Remove unused isWarmupComplete() from IngestionEngine - Add comment on cachedPointerBasedLag = -1 explaining sentinel value Signed-off-by: Kaustubh Butte --- CHANGELOG.md | 1 - .../org/opensearch/cluster/metadata/IngestionSource.java | 3 +-- .../java/org/opensearch/index/engine/IngestionEngine.java | 7 ------- .../indices/pollingingest/DefaultStreamPoller.java | 2 +- 4 files changed, 2 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d73d368ebfdee..eedd5d6ea09dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,7 +65,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Delegate getMin/getMax methods for ExitableTerms ([#20775](https://github.com/opensearch-project/OpenSearch/pull/20775)) - Fix terms lookup subquery fetch limit reading from non-existent index setting instead of cluster `max_clause_count` ([#20823](https://github.com/opensearch-project/OpenSearch/pull/20823)) - Fix array_index_out_of_bounds_exception with wildcard and aggregations ([#20842](https://github.com/opensearch-project/OpenSearch/pull/20842)) -- - Delegate getMin/getMax methods for ExitableTerms ([#20775](https://github.com/opensearch-project/OpenSearch/pull/20775)) ### Dependencies - Bump shadow-gradle-plugin from 8.3.9 to 9.3.1 ([#20569](https://github.com/opensearch-project/OpenSearch/pull/20569)) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java index ff11df9d3ca1d..3b204b2cbe348 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java @@ -8,7 +8,6 @@ package org.opensearch.cluster.metadata; -import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -270,7 +269,7 @@ public String toString() { * When warmup is enabled, shards will wait for lag to catch up before serving queries * after node restart or shard relocation. */ - @ExperimentalApi + @PublicApi(since = "3.6.0") public static class WarmupConfig { private final boolean enabled; private final TimeValue timeout; diff --git a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java index a667caa476fd8..5c3bf8b61b786 100644 --- a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java @@ -664,13 +664,6 @@ public ShardIngestionState getIngestionState() { ); } - /** - * Returns true if the warmup phase is complete and the shard is ready to serve. - */ - public boolean isWarmupComplete() { - return streamPoller.isWarmupComplete(); - } - /** * Block until warmup is complete or timeout occurs. * This method handles all warmup logic internally including timeout handling and error throwing. diff --git a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java index af0037fe3a22a..bf97954015bc8 100644 --- a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java +++ b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java @@ -59,7 +59,7 @@ public class DefaultStreamPoller implements StreamPoller { private volatile boolean reinitializeConsumer; private volatile long lastPolledMessageTimestamp = 0; - private volatile long cachedPointerBasedLag = -1; + private volatile long cachedPointerBasedLag = -1; // -1 indicates poller has not consumed any message yet private volatile long lastPointerBasedLagUpdateTime = 0; // Warmup configuration and state From e7895db7f2fbf4ac4091601e98030db99a8d2ef4 Mon Sep 17 00:00:00 2001 From: Kaustubh Butte Date: Tue, 17 Mar 2026 11:51:42 +0530 Subject: [PATCH 11/12] Address andrross and varun review: record class, default disabled, cleanup - Convert WarmupConfig from class to record (andrross) - Change warmup default to enabled=false (varun - new feature, opt-in) - Remove separate warmup convenience methods from IngestionSource, callers now use getWarmupConfig().xyz() directly - Remove unused isWarmupFailOnTimeout() from IngestionEngine - Fix outdated javadoc: cachedPointerBasedLag is -1 not 0 - Fix CHANGELOG to only contain our entry Signed-off-by: Kaustubh Butte --- CHANGELOG.md | 1 + .../cluster/metadata/IndexMetadata.java | 2 +- .../cluster/metadata/IngestionSource.java | 93 ++----------------- .../index/engine/IngestionEngine.java | 13 +-- .../pollingingest/DefaultStreamPoller.java | 18 ++-- .../metadata/IngestionSourceTests.java | 44 ++++----- 6 files changed, 46 insertions(+), 125 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eedd5d6ea09dd..78a3c93019a13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add indices to search request slowlog ([#20588](https://github.com/opensearch-project/OpenSearch/pull/20588)) - Add warmup phase to wait for lag to catch up in pull-based ingestion before serving ([#20526](https://github.com/opensearch-project/OpenSearch/pull/20526)) +- Add warmup phase to wait for lag to catch up in pull-based ingestion before serving ([#20526](https://github.com/opensearch-project/OpenSearch/pull/20526)) ### Changed - Make telemetry `Tags` immutable ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788)) - Move Randomness from server to libs/common ([#20570](https://github.com/opensearch-project/OpenSearch/pull/20570)) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 55fd611cac911..eb0494ff116ce 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -1038,7 +1038,7 @@ public Iterator> settings() { public static final String SETTING_INGESTION_SOURCE_WARMUP_ENABLED = "index.ingestion_source.warmup.enabled"; public static final Setting INGESTION_SOURCE_WARMUP_ENABLED_SETTING = Setting.boolSetting( SETTING_INGESTION_SOURCE_WARMUP_ENABLED, - true, + false, Property.IndexScope, Property.Final ); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java index 3b204b2cbe348..cbee59ae6e0aa 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java @@ -133,22 +133,6 @@ public WarmupConfig getWarmupConfig() { return warmupConfig; } - public boolean isWarmupEnabled() { - return warmupConfig.isEnabled(); - } - - public TimeValue getWarmupTimeout() { - return warmupConfig.getTimeout(); - } - - public long getWarmupLagThreshold() { - return warmupConfig.getLagThreshold(); - } - - public boolean isWarmupFailOnTimeout() { - return warmupConfig.isFailOnTimeout(); - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -265,69 +249,12 @@ public String toString() { } /** - * Class encapsulating the warmup configuration for pull-based ingestion. + * Record encapsulating the warmup configuration for pull-based ingestion. * When warmup is enabled, shards will wait for lag to catch up before serving queries * after node restart or shard relocation. */ @PublicApi(since = "3.6.0") - public static class WarmupConfig { - private final boolean enabled; - private final TimeValue timeout; - private final long lagThreshold; - private final boolean failOnTimeout; - - public WarmupConfig(boolean enabled, TimeValue timeout, long lagThreshold, boolean failOnTimeout) { - this.enabled = enabled; - this.timeout = timeout; - this.lagThreshold = lagThreshold; - this.failOnTimeout = failOnTimeout; - } - - public boolean isEnabled() { - return enabled; - } - - public TimeValue getTimeout() { - return timeout; - } - - public long getLagThreshold() { - return lagThreshold; - } - - public boolean isFailOnTimeout() { - return failOnTimeout; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - WarmupConfig that = (WarmupConfig) o; - return enabled == that.enabled - && lagThreshold == that.lagThreshold - && failOnTimeout == that.failOnTimeout - && Objects.equals(timeout, that.timeout); - } - - @Override - public int hashCode() { - return Objects.hash(enabled, timeout, lagThreshold, failOnTimeout); - } - - @Override - public String toString() { - return "WarmupConfig{" - + "enabled=" - + enabled - + ", timeout=" - + timeout - + ", lagThreshold=" - + lagThreshold - + ", failOnTimeout=" - + failOnTimeout - + '}'; - } + public record WarmupConfig(boolean enabled, TimeValue timeout, long lagThreshold, boolean failOnTimeout) { } /** @@ -373,10 +300,10 @@ public Builder(IngestionSource ingestionSource) { this.mapperSettings = new HashMap<>(ingestionSource.mapperSettings); // Copy warmup config WarmupConfig wc = ingestionSource.warmupConfig; - this.warmupEnabled = wc.isEnabled(); - this.warmupTimeout = wc.getTimeout(); - this.warmupLagThreshold = wc.getLagThreshold(); - this.warmupFailOnTimeout = wc.isFailOnTimeout(); + this.warmupEnabled = wc.enabled(); + this.warmupTimeout = wc.timeout(); + this.warmupLagThreshold = wc.lagThreshold(); + this.warmupFailOnTimeout = wc.failOnTimeout(); } public Builder setPointerInitReset(PointerInitReset pointerInitReset) { @@ -460,10 +387,10 @@ public Builder setWarmupFailOnTimeout(boolean warmupFailOnTimeout) { } public Builder setWarmupConfig(WarmupConfig warmupConfig) { - this.warmupEnabled = warmupConfig.isEnabled(); - this.warmupTimeout = warmupConfig.getTimeout(); - this.warmupLagThreshold = warmupConfig.getLagThreshold(); - this.warmupFailOnTimeout = warmupConfig.isFailOnTimeout(); + this.warmupEnabled = warmupConfig.enabled(); + this.warmupTimeout = warmupConfig.timeout(); + this.warmupLagThreshold = warmupConfig.lagThreshold(); + this.warmupFailOnTimeout = warmupConfig.failOnTimeout(); return this; } diff --git a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java index 5c3bf8b61b786..84cabd3d33f90 100644 --- a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java @@ -673,15 +673,15 @@ public ShardIngestionState getIngestionState() { */ public void awaitWarmupComplete() throws InterruptedException { IngestionSource ingestionSource = engineConfig.getIndexSettings().getIndexMetadata().getIngestionSource(); - if (ingestionSource == null || !ingestionSource.isWarmupEnabled() || streamPoller.isPaused()) { + if (ingestionSource == null || !ingestionSource.getWarmupConfig().enabled() || streamPoller.isPaused()) { return; } - long timeoutMs = ingestionSource.getWarmupTimeout().millis(); + long timeoutMs = ingestionSource.getWarmupConfig().timeout().millis(); boolean completed = streamPoller.awaitWarmupComplete(timeoutMs); if (!completed) { - if (isWarmupFailOnTimeout()) { + if (ingestionSource.getWarmupConfig().failOnTimeout()) { throw new OpenSearchException( "Ingestion warmup timed out for shard after " + timeoutMs @@ -698,11 +698,4 @@ public void awaitWarmupComplete() throws InterruptedException { } } - /** - * Returns true if shard initialization should fail when warmup times out. - */ - public boolean isWarmupFailOnTimeout() { - IngestionSource ingestionSource = engineConfig.getIndexSettings().getIndexMetadata().getIngestionSource(); - return ingestionSource != null && ingestionSource.isWarmupFailOnTimeout(); - } } diff --git a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java index bf97954015bc8..000fc7124afc6 100644 --- a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java +++ b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java @@ -180,7 +180,7 @@ private DefaultStreamPoller( // handle initial poller states this.paused = initialState == State.PAUSED; // If warmup is disabled, mark as complete immediately - if (!warmupConfig.isEnabled()) { + if (!warmupConfig.enabled()) { this.warmupComplete = true; } } @@ -213,7 +213,7 @@ protected void startPoll() { logger.info("Starting poller for shard {}", shardId); // Initialize warmup if enabled - if (warmupConfig.isEnabled() && !warmupComplete) { + if (warmupConfig.enabled() && !warmupComplete) { warmupStartTime = System.currentTimeMillis(); state = State.WARMING_UP; logger.info("Starting warmup phase for index {} shard {}, waiting for lag to catch up", indexName, shardId); @@ -237,7 +237,7 @@ protected void startPoll() { updatePointerBasedLagIfNeeded(); // Check warmup status if not yet complete - if (!warmupComplete && warmupConfig.isEnabled()) { + if (!warmupComplete && warmupConfig.enabled()) { updateWarmupStatus(); } @@ -386,7 +386,7 @@ public boolean isClosed() { @Override public boolean isWarmupComplete() { - return warmupComplete || !warmupConfig.isEnabled(); + return warmupComplete || !warmupConfig.enabled(); } /** @@ -416,12 +416,12 @@ private void setStateWithWarmupAwareness(State newState) { * Returns true if shard initialization should fail when warmup times out. */ public boolean isWarmupFailOnTimeout() { - return warmupConfig.isFailOnTimeout(); + return warmupConfig.failOnTimeout(); } @Override public boolean awaitWarmupComplete(long timeoutMs) throws InterruptedException { - if (!warmupConfig.isEnabled() || isWarmupComplete()) { + if (!warmupConfig.enabled() || isWarmupComplete()) { return true; } @@ -443,7 +443,7 @@ public boolean awaitWarmupComplete(long timeoutMs) throws InterruptedException { * Warmup uses offset-based lag (cachedPointerBasedLag) which tracks the difference between * the current consumer position and the end of the stream. This is the preferred mode for * Kafka and other sources that support offset-based lag calculation. - * Note: cachedPointerBasedLag is 0 by default and is only updated after updatePointerBasedLagIfNeeded() + * Note: cachedPointerBasedLag is -1 by default (indicating no messages consumed yet) and is only updated after updatePointerBasedLagIfNeeded() * is called. */ private void updateWarmupStatus() { @@ -456,11 +456,11 @@ private void updateWarmupStatus() { } long currentLag = cachedPointerBasedLag; - long threshold = warmupConfig.getLagThreshold(); + long threshold = warmupConfig.lagThreshold(); long elapsedTime = System.currentTimeMillis() - warmupStartTime; boolean lagBelowThreshold = currentLag >= 0 && currentLag <= threshold; - boolean timeoutReached = elapsedTime >= warmupConfig.getTimeout().millis(); + boolean timeoutReached = elapsedTime >= warmupConfig.timeout().millis(); if (lagBelowThreshold) { warmupComplete = true; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java index f6e8c096f2a88..05f63adc9c931 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java @@ -108,7 +108,7 @@ public void testToString() { .setErrorStrategy(DROP) .build(); String expected = - "IngestionSource{type='type',pointer_init_reset='PointerInitReset{type='RESET_BY_OFFSET', value=1000}',error_strategy='DROP', params={key=value}, maxPollSize=1000, pollTimeout=1000, numProcessorThreads=1, blockingQueueSize=100, allActiveIngestion=false, pointerBasedLagUpdateInterval=10s, mapperType='DEFAULT', mapperSettings={}, warmupConfig=WarmupConfig{enabled=true, timeout=5m, lagThreshold=100, failOnTimeout=false}}"; + "IngestionSource{type='type',pointer_init_reset='PointerInitReset{type='RESET_BY_OFFSET', value=1000}',error_strategy='DROP', params={key=value}, maxPollSize=1000, pollTimeout=1000, numProcessorThreads=1, blockingQueueSize=100, allActiveIngestion=false, pointerBasedLagUpdateInterval=10s, mapperType='DEFAULT', mapperSettings={}, warmupConfig=WarmupConfig[enabled=false, timeout=5m, lagThreshold=100, failOnTimeout=false]}"; assertEquals(expected, source.toString()); } @@ -192,20 +192,20 @@ public void testWarmupConfigurationConstructorAndGetters() { .setWarmupFailOnTimeout(true) .build(); - assertTrue("Warmup should be enabled", source.isWarmupEnabled()); - assertEquals(TimeValue.timeValueMinutes(10), source.getWarmupTimeout()); - assertEquals(100, source.getWarmupLagThreshold()); - assertTrue("Should fail on timeout", source.isWarmupFailOnTimeout()); + assertTrue("Warmup should be enabled", source.getWarmupConfig().enabled()); + assertEquals(TimeValue.timeValueMinutes(10), source.getWarmupConfig().timeout()); + assertEquals(100, source.getWarmupConfig().lagThreshold()); + assertTrue("Should fail on timeout", source.getWarmupConfig().failOnTimeout()); } public void testWarmupConfigurationDefaults() { // Test default warmup values IngestionSource source = new IngestionSource.Builder("type").build(); - assertTrue("Warmup should be enabled by default", source.isWarmupEnabled()); - assertEquals(TimeValue.timeValueMinutes(5), source.getWarmupTimeout()); - assertEquals(100, source.getWarmupLagThreshold()); - assertFalse("Should not fail on timeout by default", source.isWarmupFailOnTimeout()); + assertFalse("Warmup should be disabled by default", source.getWarmupConfig().enabled()); + assertEquals(TimeValue.timeValueMinutes(5), source.getWarmupConfig().timeout()); + assertEquals(100, source.getWarmupConfig().lagThreshold()); + assertFalse("Should not fail on timeout by default", source.getWarmupConfig().failOnTimeout()); } public void testWarmupConfigurationEquality() { @@ -244,19 +244,19 @@ public void testWarmupConfigurationCopiedByBuilder() { // Create a copy using the copy constructor IngestionSource copy = new IngestionSource.Builder(original).build(); - assertEquals(original.isWarmupEnabled(), copy.isWarmupEnabled()); - assertEquals(original.getWarmupTimeout(), copy.getWarmupTimeout()); - assertEquals(original.getWarmupLagThreshold(), copy.getWarmupLagThreshold()); - assertEquals(original.isWarmupFailOnTimeout(), copy.isWarmupFailOnTimeout()); + assertEquals(original.getWarmupConfig().enabled(), copy.getWarmupConfig().enabled()); + assertEquals(original.getWarmupConfig().timeout(), copy.getWarmupConfig().timeout()); + assertEquals(original.getWarmupConfig().lagThreshold(), copy.getWarmupConfig().lagThreshold()); + assertEquals(original.getWarmupConfig().failOnTimeout(), copy.getWarmupConfig().failOnTimeout()); } public void testWarmupConfigClass() { IngestionSource.WarmupConfig config1 = new IngestionSource.WarmupConfig(true, TimeValue.timeValueMinutes(10), 100, true); - assertEquals(true, config1.isEnabled()); - assertEquals(TimeValue.timeValueMinutes(10), config1.getTimeout()); - assertEquals(100, config1.getLagThreshold()); - assertEquals(true, config1.isFailOnTimeout()); + assertEquals(true, config1.enabled()); + assertEquals(TimeValue.timeValueMinutes(10), config1.timeout()); + assertEquals(100, config1.lagThreshold()); + assertEquals(true, config1.failOnTimeout()); // Test equals and hashCode IngestionSource.WarmupConfig config2 = new IngestionSource.WarmupConfig(true, TimeValue.timeValueMinutes(10), 100, true); @@ -268,7 +268,7 @@ public void testWarmupConfigClass() { assertNotEquals(config1, config3); // Test toString - String expectedToString = "WarmupConfig{enabled=true, timeout=10m, lagThreshold=100, failOnTimeout=true}"; + String expectedToString = "WarmupConfig[enabled=true, timeout=10m, lagThreshold=100, failOnTimeout=true]"; assertEquals(expectedToString, config1.toString()); } @@ -278,9 +278,9 @@ public void testSetWarmupConfig() { IngestionSource source = new IngestionSource.Builder("type").setWarmupConfig(warmupConfig).build(); assertEquals(warmupConfig, source.getWarmupConfig()); - assertTrue(source.isWarmupEnabled()); - assertEquals(TimeValue.timeValueMinutes(15), source.getWarmupTimeout()); - assertEquals(200, source.getWarmupLagThreshold()); - assertTrue(source.isWarmupFailOnTimeout()); + assertTrue(source.getWarmupConfig().enabled()); + assertEquals(TimeValue.timeValueMinutes(15), source.getWarmupConfig().timeout()); + assertEquals(200, source.getWarmupConfig().lagThreshold()); + assertTrue(source.getWarmupConfig().failOnTimeout()); } } From 911ce7f6c4e346a51406a7e7af8d17a9abf82d41 Mon Sep 17 00:00:00 2001 From: Kaustubh Butte Date: Thu, 19 Mar 2026 01:20:47 +0530 Subject: [PATCH 12/12] Simplify warmup settings: remove enabled and fail_on_timeout Per andrross review feedback: - Remove warmup.enabled setting: use timeout=-1 (disabled) vs timeout>=0 (enabled) - Remove warmup.fail_on_timeout setting: always proceed with warning on timeout to avoid shard init failure loops (livelock risk) - WarmupConfig record now has only 2 fields: timeout and lagThreshold - Default: warmup disabled (timeout=-1), opt-in by setting a positive timeout - Adapt to upstream Indexer refactor (getEngine -> getIndexer/EngineBackedIndexer) Signed-off-by: Kaustubh Butte --- CHANGELOG.md | 7 -- .../plugin/kafka/IngestFromKafkaIT.java | 14 --- .../plugin/kafka/KafkaIngestionBaseIT.java | 1 - .../plugin/kafka/RemoteStoreKafkaIT.java | 8 -- .../plugin/kafka/KafkaSingleNodeTests.java | 3 - .../cluster/metadata/IndexMetadata.java | 34 ++------ .../cluster/metadata/IngestionSource.java | 32 +++---- .../common/settings/IndexScopedSettings.java | 2 - .../index/engine/IngestionEngine.java | 20 +---- .../opensearch/index/shard/IndexShard.java | 11 ++- .../pollingingest/DefaultStreamPoller.java | 24 ++---- .../metadata/IngestionSourceTests.java | 47 ++++------ .../DefaultStreamPollerTests.java | 86 +++++-------------- 13 files changed, 70 insertions(+), 219 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78a3c93019a13..84b598fd7a9f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,13 +23,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add intra segment support for single-value metric aggregations ([#20503](https://github.com/opensearch-project/OpenSearch/pull/20503)) - Add ref_path support for package-based hunspell dictionary loading ([#20840](https://github.com/opensearch-project/OpenSearch/pull/20840)) - Add support for enabling pluggable data formats, starting with phase-1 of decoupling shard from engine, and introducing basic abstractions ([#20675](https://github.com/opensearch-project/OpenSearch/pull/20675)) -- Add getWrappedScorer method to ProfileScorer for plugin access to wrapped scorers ([#20548](https://github.com/opensearch-project/OpenSearch/issues/20548)) -- Support expected cluster name with validation in CCS Sniff mode ([#20532](https://github.com/opensearch-project/OpenSearch/pull/20532)) -- Add security policy to allow `accessUnixDomainSocket` in `transport-grpc` module ([#20463](https://github.com/opensearch-project/OpenSearch/pull/20463)) -- Add range validations in query builder and field mapper ([#20497](https://github.com/opensearch-project/OpenSearch/issues/20497)) -- [Workload Management] Enhance Scroll API support for autotagging ([#20151](https://github.com/opensearch-project/OpenSearch/pull/20151)) -- Add indices to search request slowlog ([#20588](https://github.com/opensearch-project/OpenSearch/pull/20588)) -- Add warmup phase to wait for lag to catch up in pull-based ingestion before serving ([#20526](https://github.com/opensearch-project/OpenSearch/pull/20526)) - Add warmup phase to wait for lag to catch up in pull-based ingestion before serving ([#20526](https://github.com/opensearch-project/OpenSearch/pull/20526)) ### Changed diff --git a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java index d5ae03f69f7f8..e0958b32f0bbc 100644 --- a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java +++ b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/IngestFromKafkaIT.java @@ -107,7 +107,6 @@ public void testKafkaIngestion_RewindByTimeStamp() { .put("ingestion_source.param.topic", "test") .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "latest") - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -136,7 +135,6 @@ public void testKafkaIngestion_RewindByOffset() { .put("ingestion_source.param.topic", "test") .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "latest") - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -272,7 +270,6 @@ public void testAllActiveIngestion() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.pointer.init.reset", "earliest") - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -372,7 +369,6 @@ public void testReplicaPromotionOnAllActiveIngestion() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.pointer.init.reset", "earliest") - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -425,7 +421,6 @@ public void testSnapshotRestoreOnAllActiveIngestion() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.pointer.init.reset", "earliest") - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -507,7 +502,6 @@ public void testResetPollerInAllActiveIngestion() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.pointer.init.reset", "earliest") - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -548,7 +542,6 @@ public void testAllActiveOffsetBasedLag() throws Exception { .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.pointer_based_lag_update_interval", "3s") - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -620,7 +613,6 @@ public void testAllActiveIngestionBatchStartPointerOnReplicaPromotion() throws E .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -715,7 +707,6 @@ public void testAllActiveIngestionPeriodicFlush() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .put("index.periodic_flush_interval", "5s") .build(), @@ -751,7 +742,6 @@ public void testRawPayloadMapperIngestion() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.mapper_type", "raw_payload") .put("ingestion_source.error_strategy", "drop") - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -869,7 +859,6 @@ public void testDynamicUpdateKafkaParams() throws Exception { .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "latest") .put("ingestion_source.param.max.poll.records", "100") - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), mapping @@ -980,7 +969,6 @@ public void testConsumerInitializationFailureAndRecovery() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "none") - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), mapping @@ -1061,7 +1049,6 @@ public void testDynamicConfigUpdateOnNoMessages() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "earliest") - .put("ingestion_source.warmup.enabled", false) .put("ingestion_source.all_active", true) .build(), mapping @@ -1372,7 +1359,6 @@ public void testWarmupPhase() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) - .put("ingestion_source.warmup.enabled", true) .put("ingestion_source.warmup.lag_threshold", 0) .put("ingestion_source.warmup.timeout", "10m") .put("ingestion_source.all_active", true) diff --git a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/KafkaIngestionBaseIT.java b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/KafkaIngestionBaseIT.java index 7c57c74e425ee..604df7a7c8414 100644 --- a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/KafkaIngestionBaseIT.java +++ b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/KafkaIngestionBaseIT.java @@ -205,7 +205,6 @@ protected void createIndexWithDefaultSettings(String indexName, int numShards, i // set custom kafka consumer properties .put("ingestion_source.param.fetch.min.bytes", 30000) .put("ingestion_source.param.enable.auto.commit", false) - .put("ingestion_source.warmup.enabled", false) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" ); diff --git a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/RemoteStoreKafkaIT.java b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/RemoteStoreKafkaIT.java index 5c712dd406341..1d6f49e473bc0 100644 --- a/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/RemoteStoreKafkaIT.java +++ b/plugins/ingestion-kafka/src/internalClusterTest/java/org/opensearch/plugin/kafka/RemoteStoreKafkaIT.java @@ -157,7 +157,6 @@ public void testErrorStrategy() throws Exception { .put("ingestion_source.internal_queue_size", "1000") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) - .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -399,7 +398,6 @@ public void testExternalVersioningWithDisabledGCDeletes() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) - .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .put("index.gc_deletes", "0") .build(), @@ -512,7 +510,6 @@ public void testOffsetUpdateOnBlockErrorPolicy() throws Exception { .put("ingestion_source.internal_queue_size", "1000") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) - .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -558,7 +555,6 @@ public void testConsumerResetByTimestamp() throws Exception { .put("ingestion_source.internal_queue_size", "1000") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) - .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -677,7 +673,6 @@ public void testIndexRelocation() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "earliest") - .put("ingestion_source.warmup.enabled", false) .put("index.routing.allocation.require._name", nodeA) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -729,7 +724,6 @@ public void testKafkaConnectionLost() throws Exception { .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("ingestion_source.param.auto.offset.reset", "earliest") - .put("ingestion_source.warmup.enabled", false) .put("index.routing.allocation.require._name", nodeA) .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -793,7 +787,6 @@ public void testBatchStartPointerOnReplicaPromotion() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) - .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .build(), "{\"properties\":{\"name\":{\"type\": \"text\"},\"age\":{\"type\": \"integer\"}}}}" @@ -857,7 +850,6 @@ public void testPeriodicFlush() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", topicName) .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) - .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .put("index.periodic_flush_interval", "5s") .build(), diff --git a/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java b/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java index 9a0990d7e1728..2aa0c81731698 100644 --- a/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java +++ b/plugins/ingestion-kafka/src/test/java/org/opensearch/plugin/kafka/KafkaSingleNodeTests.java @@ -81,7 +81,6 @@ public void testPauseAndResumeAPIs() throws Exception { .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) .put("index.replication.type", "SEGMENT") .put("ingestion_source.pointer_based_lag_update_interval", "0") - .put("ingestion_source.warmup.enabled", false) .build(), mappings ); @@ -152,7 +151,6 @@ public void testShardInitializationUsingUnknownTopic() throws Exception { .put("ingestion_source.pointer.init.reset", "earliest") .put("ingestion_source.param.topic", "unknownTopic") .put("ingestion_source.param.bootstrap_servers", kafka.getBootstrapServers()) - .put("ingestion_source.warmup.enabled", false) .put("index.replication.type", "SEGMENT") .build(), mappings @@ -176,7 +174,6 @@ public void testConsumerSettingUpdateWithMultipleProcessorThreads() throws Excep .put("ingestion_source.param.auto.offset.reset", "none") .put("ingestion_source.num_processor_threads", 5) .put("index.replication.type", "SEGMENT") - .put("ingestion_source.warmup.enabled", false) .build(), mappings ); diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index eb0494ff116ce..eb528115bbc02 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -1031,25 +1031,15 @@ public Iterator> settings() { }, Property.IndexScope, Property.Final) ); - /** - * Defines if warmup phase is enabled for pull-based ingestion. When enabled, shards will wait for - * lag to catch up before serving queries after node restart or shard relocation. - */ - public static final String SETTING_INGESTION_SOURCE_WARMUP_ENABLED = "index.ingestion_source.warmup.enabled"; - public static final Setting INGESTION_SOURCE_WARMUP_ENABLED_SETTING = Setting.boolSetting( - SETTING_INGESTION_SOURCE_WARMUP_ENABLED, - false, - Property.IndexScope, - Property.Final - ); - /** * Defines the maximum time to wait for lag to catch up during warmup phase. + * A value of -1 means warmup is disabled (the default). A value >= 0 enables warmup with that timeout. */ public static final String SETTING_INGESTION_SOURCE_WARMUP_TIMEOUT = "index.ingestion_source.warmup.timeout"; - public static final Setting INGESTION_SOURCE_WARMUP_TIMEOUT_SETTING = Setting.positiveTimeSetting( + public static final Setting INGESTION_SOURCE_WARMUP_TIMEOUT_SETTING = Setting.timeSetting( SETTING_INGESTION_SOURCE_WARMUP_TIMEOUT, - new TimeValue(5, TimeUnit.MINUTES), + TimeValue.timeValueMillis(-1), + TimeValue.timeValueMillis(-1), Property.IndexScope, Property.Final ); @@ -1067,18 +1057,6 @@ public Iterator> settings() { Property.Final ); - /** - * Defines if shard initialization should fail when warmup times out. - * If false, shard proceeds with a warning. If true, shard initialization fails. - */ - public static final String SETTING_INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT = "index.ingestion_source.warmup.fail_on_timeout"; - public static final Setting INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT_SETTING = Setting.boolSetting( - SETTING_INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT, - false, - Property.IndexScope, - Property.Final - ); - /** * an internal index format description, allowing us to find out if this index is upgraded or needs upgrading */ @@ -1351,10 +1329,8 @@ public IngestionSource getIngestionSource() { // Warmup settings final IngestionSource.WarmupConfig warmupConfig = new IngestionSource.WarmupConfig( - INGESTION_SOURCE_WARMUP_ENABLED_SETTING.get(settings), INGESTION_SOURCE_WARMUP_TIMEOUT_SETTING.get(settings), - INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING.get(settings), - INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT_SETTING.get(settings) + INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING.get(settings) ); return new IngestionSource.Builder(ingestionSourceType).setParams(ingestionSourceParams) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java index cbee59ae6e0aa..6b1400c305e08 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IngestionSource.java @@ -27,8 +27,6 @@ import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_NUM_PROCESSOR_THREADS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_POINTER_BASED_LAG_UPDATE_INTERVAL_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_POLL_TIMEOUT; -import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_WARMUP_ENABLED_SETTING; -import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INGESTION_SOURCE_WARMUP_TIMEOUT_SETTING; @@ -250,11 +248,17 @@ public String toString() { /** * Record encapsulating the warmup configuration for pull-based ingestion. - * When warmup is enabled, shards will wait for lag to catch up before serving queries - * after node restart or shard relocation. + * When warmup is enabled (timeout >= 0), shards will wait for lag to catch up before serving queries + * after node restart or shard relocation. A timeout of -1 means warmup is disabled. */ @PublicApi(since = "3.6.0") - public record WarmupConfig(boolean enabled, TimeValue timeout, long lagThreshold, boolean failOnTimeout) { + public record WarmupConfig(TimeValue timeout, long lagThreshold) { + /** + * Returns true if warmup is enabled (timeout >= 0). + */ + public boolean isEnabled() { + return timeout.millis() >= 0; + } } /** @@ -278,10 +282,8 @@ public static class Builder { private IngestionMessageMapper.MapperType mapperType = INGESTION_SOURCE_MAPPER_TYPE_SETTING.getDefault(Settings.EMPTY); private Map mapperSettings = new HashMap<>(); // Warmup configuration - private boolean warmupEnabled = INGESTION_SOURCE_WARMUP_ENABLED_SETTING.getDefault(Settings.EMPTY); private TimeValue warmupTimeout = INGESTION_SOURCE_WARMUP_TIMEOUT_SETTING.getDefault(Settings.EMPTY); private long warmupLagThreshold = INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING.getDefault(Settings.EMPTY); - private boolean warmupFailOnTimeout = INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT_SETTING.getDefault(Settings.EMPTY); public Builder(String type) { this.type = type; @@ -300,10 +302,8 @@ public Builder(IngestionSource ingestionSource) { this.mapperSettings = new HashMap<>(ingestionSource.mapperSettings); // Copy warmup config WarmupConfig wc = ingestionSource.warmupConfig; - this.warmupEnabled = wc.enabled(); this.warmupTimeout = wc.timeout(); this.warmupLagThreshold = wc.lagThreshold(); - this.warmupFailOnTimeout = wc.failOnTimeout(); } public Builder setPointerInitReset(PointerInitReset pointerInitReset) { @@ -366,11 +366,6 @@ public Builder setMapperSettings(Map mapperSettings) { return this; } - public Builder setWarmupEnabled(boolean warmupEnabled) { - this.warmupEnabled = warmupEnabled; - return this; - } - public Builder setWarmupTimeout(TimeValue warmupTimeout) { this.warmupTimeout = warmupTimeout; return this; @@ -381,21 +376,14 @@ public Builder setWarmupLagThreshold(long warmupLagThreshold) { return this; } - public Builder setWarmupFailOnTimeout(boolean warmupFailOnTimeout) { - this.warmupFailOnTimeout = warmupFailOnTimeout; - return this; - } - public Builder setWarmupConfig(WarmupConfig warmupConfig) { - this.warmupEnabled = warmupConfig.enabled(); this.warmupTimeout = warmupConfig.timeout(); this.warmupLagThreshold = warmupConfig.lagThreshold(); - this.warmupFailOnTimeout = warmupConfig.failOnTimeout(); return this; } public IngestionSource build() { - WarmupConfig warmupConfig = new WarmupConfig(warmupEnabled, warmupTimeout, warmupLagThreshold, warmupFailOnTimeout); + WarmupConfig warmupConfig = new WarmupConfig(warmupTimeout, warmupLagThreshold); return new IngestionSource( type, pointerInitReset, diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index d589b39fe81b9..e242304e446b5 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -292,10 +292,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexMetadata.INGESTION_SOURCE_POINTER_BASED_LAG_UPDATE_INTERVAL_SETTING, IndexMetadata.INGESTION_SOURCE_MAPPER_TYPE_SETTING, IndexMetadata.INGESTION_SOURCE_MAPPER_SETTINGS, - IndexMetadata.INGESTION_SOURCE_WARMUP_ENABLED_SETTING, IndexMetadata.INGESTION_SOURCE_WARMUP_TIMEOUT_SETTING, IndexMetadata.INGESTION_SOURCE_WARMUP_LAG_THRESHOLD_SETTING, - IndexMetadata.INGESTION_SOURCE_WARMUP_FAIL_ON_TIMEOUT_SETTING, // Settings for search replica IndexMetadata.INDEX_NUMBER_OF_SEARCH_REPLICAS_SETTING, diff --git a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java index 84cabd3d33f90..ebcbd776135cd 100644 --- a/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/IngestionEngine.java @@ -666,14 +666,13 @@ public ShardIngestionState getIngestionState() { /** * Block until warmup is complete or timeout occurs. - * This method handles all warmup logic internally including timeout handling and error throwing. + * This method handles all warmup logic internally. On timeout, always logs a warning and proceeds. * - * @throws OpenSearchException if warmup times out and failOnTimeout is configured * @throws InterruptedException if the thread is interrupted while waiting */ public void awaitWarmupComplete() throws InterruptedException { IngestionSource ingestionSource = engineConfig.getIndexSettings().getIndexMetadata().getIngestionSource(); - if (ingestionSource == null || !ingestionSource.getWarmupConfig().enabled() || streamPoller.isPaused()) { + if (ingestionSource == null || !ingestionSource.getWarmupConfig().isEnabled() || streamPoller.isPaused()) { return; } @@ -681,20 +680,7 @@ public void awaitWarmupComplete() throws InterruptedException { boolean completed = streamPoller.awaitWarmupComplete(timeoutMs); if (!completed) { - if (ingestionSource.getWarmupConfig().failOnTimeout()) { - throw new OpenSearchException( - "Ingestion warmup timed out for shard after " - + timeoutMs - + "ms. " - + "Configure warmup.fail_on_timeout=false to proceed with stale data." - ); - } - // Log warning when proceeding despite timeout - logger.warn( - "Ingestion warmup timed out for shard after {}ms, proceeding with potentially stale data. " - + "Set warmup.fail_on_timeout=true to fail shard initialization on timeout.", - timeoutMs - ); + logger.warn("Ingestion warmup timed out for shard after {}ms, proceeding with potentially stale data.", timeoutMs); } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index ae8a50fda5c93..1c155c897acba 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -2489,8 +2489,7 @@ public void postRecovery(String reason) throws IndexShardStartedException, Index getIndexer().refresh("post_recovery"); // Wait for ingestion warmup if enabled (pull-based ingestion only) - Engine engine = getEngine(); - handlePullBasedIngestionWarmup(engine); + handlePullBasedIngestionWarmup(getIndexer()); synchronized (mutex) { if (state == IndexShardState.CLOSED) { @@ -2510,9 +2509,13 @@ public void postRecovery(String reason) throws IndexShardStartedException, Index * When warmup is enabled, this method blocks until the shard has caught up with the streaming source * or until the configured timeout is reached. * - * @param engine the engine to check for warmup + * @param indexer the indexer to check for warmup */ - private void handlePullBasedIngestionWarmup(Engine engine) { + private void handlePullBasedIngestionWarmup(Indexer indexer) { + if (!(indexer instanceof EngineBackedIndexer)) { + return; + } + Engine engine = ((EngineBackedIndexer) indexer).getEngine(); if (engine instanceof IngestionEngine) { IngestionEngine ingestionEngine = (IngestionEngine) engine; try { diff --git a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java index 000fc7124afc6..4dc51e0539d83 100644 --- a/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java +++ b/server/src/main/java/org/opensearch/indices/pollingingest/DefaultStreamPoller.java @@ -180,7 +180,7 @@ private DefaultStreamPoller( // handle initial poller states this.paused = initialState == State.PAUSED; // If warmup is disabled, mark as complete immediately - if (!warmupConfig.enabled()) { + if (!warmupConfig.isEnabled()) { this.warmupComplete = true; } } @@ -213,7 +213,7 @@ protected void startPoll() { logger.info("Starting poller for shard {}", shardId); // Initialize warmup if enabled - if (warmupConfig.enabled() && !warmupComplete) { + if (warmupConfig.isEnabled() && !warmupComplete) { warmupStartTime = System.currentTimeMillis(); state = State.WARMING_UP; logger.info("Starting warmup phase for index {} shard {}, waiting for lag to catch up", indexName, shardId); @@ -237,7 +237,7 @@ protected void startPoll() { updatePointerBasedLagIfNeeded(); // Check warmup status if not yet complete - if (!warmupComplete && warmupConfig.enabled()) { + if (!warmupComplete && warmupConfig.isEnabled()) { updateWarmupStatus(); } @@ -386,7 +386,7 @@ public boolean isClosed() { @Override public boolean isWarmupComplete() { - return warmupComplete || !warmupConfig.enabled(); + return warmupComplete || !warmupConfig.isEnabled(); } /** @@ -412,16 +412,9 @@ private void setStateWithWarmupAwareness(State newState) { this.state = newState; } - /** - * Returns true if shard initialization should fail when warmup times out. - */ - public boolean isWarmupFailOnTimeout() { - return warmupConfig.failOnTimeout(); - } - @Override public boolean awaitWarmupComplete(long timeoutMs) throws InterruptedException { - if (!warmupConfig.enabled() || isWarmupComplete()) { + if (!warmupConfig.isEnabled() || isWarmupComplete()) { return true; } @@ -740,12 +733,7 @@ public static class Builder { private IngestionMessageMapper.MapperType mapperType = IngestionMessageMapper.MapperType.DEFAULT; private Map mapperSettings = Collections.emptyMap(); // Warmup configuration - default matches IndexMetadata settings - private IngestionSource.WarmupConfig warmupConfig = new IngestionSource.WarmupConfig( - false, - TimeValue.timeValueMinutes(5), - 100L, - false - ); + private IngestionSource.WarmupConfig warmupConfig = new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 100L); /** * Initialize the builder with mandatory parameters diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java index 05f63adc9c931..f9a1ec9bb3f7d 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/IngestionSourceTests.java @@ -108,7 +108,7 @@ public void testToString() { .setErrorStrategy(DROP) .build(); String expected = - "IngestionSource{type='type',pointer_init_reset='PointerInitReset{type='RESET_BY_OFFSET', value=1000}',error_strategy='DROP', params={key=value}, maxPollSize=1000, pollTimeout=1000, numProcessorThreads=1, blockingQueueSize=100, allActiveIngestion=false, pointerBasedLagUpdateInterval=10s, mapperType='DEFAULT', mapperSettings={}, warmupConfig=WarmupConfig[enabled=false, timeout=5m, lagThreshold=100, failOnTimeout=false]}"; + "IngestionSource{type='type',pointer_init_reset='PointerInitReset{type='RESET_BY_OFFSET', value=1000}',error_strategy='DROP', params={key=value}, maxPollSize=1000, pollTimeout=1000, numProcessorThreads=1, blockingQueueSize=100, allActiveIngestion=false, pointerBasedLagUpdateInterval=10s, mapperType='DEFAULT', mapperSettings={}, warmupConfig=WarmupConfig[timeout=-1, lagThreshold=100]}"; assertEquals(expected, source.toString()); } @@ -182,30 +182,26 @@ public void testWarmupConfigurationConstructorAndGetters() { Map params = new HashMap<>(); params.put("key", "value"); - // Test with warmup configuration + // Test with warmup configuration (timeout >= 0 means enabled) IngestionSource source = new IngestionSource.Builder("type").setParams(params) .setPointerInitReset(pointerInitReset) .setErrorStrategy(DROP) - .setWarmupEnabled(true) .setWarmupTimeout(TimeValue.timeValueMinutes(10)) .setWarmupLagThreshold(100) - .setWarmupFailOnTimeout(true) .build(); - assertTrue("Warmup should be enabled", source.getWarmupConfig().enabled()); + assertTrue("Warmup should be enabled", source.getWarmupConfig().isEnabled()); assertEquals(TimeValue.timeValueMinutes(10), source.getWarmupConfig().timeout()); assertEquals(100, source.getWarmupConfig().lagThreshold()); - assertTrue("Should fail on timeout", source.getWarmupConfig().failOnTimeout()); } public void testWarmupConfigurationDefaults() { - // Test default warmup values + // Test default warmup values (timeout=-1 means disabled) IngestionSource source = new IngestionSource.Builder("type").build(); - assertFalse("Warmup should be disabled by default", source.getWarmupConfig().enabled()); - assertEquals(TimeValue.timeValueMinutes(5), source.getWarmupConfig().timeout()); + assertFalse("Warmup should be disabled by default", source.getWarmupConfig().isEnabled()); + assertEquals(TimeValue.timeValueMillis(-1), source.getWarmupConfig().timeout()); assertEquals(100, source.getWarmupConfig().lagThreshold()); - assertFalse("Should not fail on timeout by default", source.getWarmupConfig().failOnTimeout()); } public void testWarmupConfigurationEquality() { @@ -213,21 +209,21 @@ public void testWarmupConfigurationEquality() { params.put("key", "value"); IngestionSource source1 = new IngestionSource.Builder("type").setParams(params) - .setWarmupEnabled(true) + .setWarmupTimeout(TimeValue.timeValueMinutes(10)) .setWarmupLagThreshold(100) .build(); IngestionSource source2 = new IngestionSource.Builder("type").setParams(params) - .setWarmupEnabled(true) + .setWarmupTimeout(TimeValue.timeValueMinutes(10)) .setWarmupLagThreshold(100) .build(); assertEquals(source1, source2); assertEquals(source1.hashCode(), source2.hashCode()); - // Test inequality with different warmup settings + // Test inequality with different warmup settings (disabled vs enabled) IngestionSource source3 = new IngestionSource.Builder("type").setParams(params) - .setWarmupEnabled(false) + .setWarmupTimeout(TimeValue.timeValueMillis(-1)) .setWarmupLagThreshold(100) .build(); @@ -235,52 +231,47 @@ public void testWarmupConfigurationEquality() { } public void testWarmupConfigurationCopiedByBuilder() { - IngestionSource original = new IngestionSource.Builder("type").setWarmupEnabled(true) - .setWarmupTimeout(TimeValue.timeValueMinutes(10)) + IngestionSource original = new IngestionSource.Builder("type").setWarmupTimeout(TimeValue.timeValueMinutes(10)) .setWarmupLagThreshold(500) - .setWarmupFailOnTimeout(true) .build(); // Create a copy using the copy constructor IngestionSource copy = new IngestionSource.Builder(original).build(); - assertEquals(original.getWarmupConfig().enabled(), copy.getWarmupConfig().enabled()); assertEquals(original.getWarmupConfig().timeout(), copy.getWarmupConfig().timeout()); assertEquals(original.getWarmupConfig().lagThreshold(), copy.getWarmupConfig().lagThreshold()); - assertEquals(original.getWarmupConfig().failOnTimeout(), copy.getWarmupConfig().failOnTimeout()); } public void testWarmupConfigClass() { - IngestionSource.WarmupConfig config1 = new IngestionSource.WarmupConfig(true, TimeValue.timeValueMinutes(10), 100, true); + IngestionSource.WarmupConfig config1 = new IngestionSource.WarmupConfig(TimeValue.timeValueMinutes(10), 100); - assertEquals(true, config1.enabled()); + assertTrue(config1.isEnabled()); assertEquals(TimeValue.timeValueMinutes(10), config1.timeout()); assertEquals(100, config1.lagThreshold()); - assertEquals(true, config1.failOnTimeout()); // Test equals and hashCode - IngestionSource.WarmupConfig config2 = new IngestionSource.WarmupConfig(true, TimeValue.timeValueMinutes(10), 100, true); + IngestionSource.WarmupConfig config2 = new IngestionSource.WarmupConfig(TimeValue.timeValueMinutes(10), 100); assertEquals(config1, config2); assertEquals(config1.hashCode(), config2.hashCode()); // Test inequality - IngestionSource.WarmupConfig config3 = new IngestionSource.WarmupConfig(false, TimeValue.timeValueMinutes(10), 100, true); + IngestionSource.WarmupConfig config3 = new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 100); assertNotEquals(config1, config3); + assertFalse(config3.isEnabled()); // Test toString - String expectedToString = "WarmupConfig[enabled=true, timeout=10m, lagThreshold=100, failOnTimeout=true]"; + String expectedToString = "WarmupConfig[timeout=10m, lagThreshold=100]"; assertEquals(expectedToString, config1.toString()); } public void testSetWarmupConfig() { - IngestionSource.WarmupConfig warmupConfig = new IngestionSource.WarmupConfig(true, TimeValue.timeValueMinutes(15), 200, true); + IngestionSource.WarmupConfig warmupConfig = new IngestionSource.WarmupConfig(TimeValue.timeValueMinutes(15), 200); IngestionSource source = new IngestionSource.Builder("type").setWarmupConfig(warmupConfig).build(); assertEquals(warmupConfig, source.getWarmupConfig()); - assertTrue(source.getWarmupConfig().enabled()); + assertTrue(source.getWarmupConfig().isEnabled()); assertEquals(TimeValue.timeValueMinutes(15), source.getWarmupConfig().timeout()); assertEquals(200, source.getWarmupConfig().lagThreshold()); - assertTrue(source.getWarmupConfig().failOnTimeout()); } } diff --git a/server/src/test/java/org/opensearch/indices/pollingingest/DefaultStreamPollerTests.java b/server/src/test/java/org/opensearch/indices/pollingingest/DefaultStreamPollerTests.java index fa0d2e09a7fe9..7979c2c86dec1 100644 --- a/server/src/test/java/org/opensearch/indices/pollingingest/DefaultStreamPollerTests.java +++ b/server/src/test/java/org/opensearch/indices/pollingingest/DefaultStreamPollerTests.java @@ -97,7 +97,7 @@ public void setUp() throws Exception { 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); partitionedBlockingQueueContainer.startProcessorThreads(); } @@ -172,7 +172,7 @@ public void testResetStateEarliest() throws InterruptedException { 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); CountDownLatch latch = new CountDownLatch(2); doAnswer(invocation -> { @@ -207,7 +207,7 @@ public void testResetStateLatest() throws InterruptedException { 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); // Set up latch to wait for 2 messages to be processed @@ -250,7 +250,7 @@ public void testResetStateRewindByOffset() throws InterruptedException { 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); CountDownLatch latch = new CountDownLatch(1); doAnswer(invocation -> { @@ -337,7 +337,7 @@ public void testDropErrorIngestionStrategy() throws TimeoutException, Interrupte 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); poller.start(); Thread.sleep(sleepTime); @@ -401,7 +401,7 @@ public void testBlockErrorIngestionStrategy() throws TimeoutException, Interrupt 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); poller.start(); Thread.sleep(sleepTime); @@ -439,7 +439,7 @@ public void testProcessingErrorWithBlockErrorIngestionStrategy() throws TimeoutE 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); poller.start(); Thread.sleep(sleepTime); @@ -513,7 +513,7 @@ public void testPersistedBatchStartPointer() throws TimeoutException, Interrupte 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); poller.start(); Thread.sleep(sleepTime); @@ -584,7 +584,7 @@ public void testConsumerInitializationRetry() throws Exception { 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); poller.start(); @@ -628,7 +628,7 @@ public void testConsumerReinitializationAfterProcessingMessages() throws Excepti 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); // Start and wait for 2 messages to be processed @@ -685,7 +685,7 @@ public void testConsumerReinitializationWithNoInitialMessages() throws Exception 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); // Start poller @@ -730,7 +730,7 @@ public void testGetBatchStartPointerWithNullInitialPointer() { 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); // When all queues return null and initialBatchStartPointer is null, getBatchStartPointer should return null @@ -741,7 +741,7 @@ public void testGetBatchStartPointerWithNullInitialPointer() { public void testWarmupDisabledBehavior() { // When warmup is disabled, isWarmupComplete should return true immediately - // The default poller in setUp has warmupEnabled=false + // The default poller in setUp has warmup disabled (timeout=-1) assertTrue(poller.isWarmupComplete()); // Warmup disabled means it's considered complete // Create another poller with warmup explicitly disabled @@ -760,7 +760,7 @@ public void testWarmupDisabledBehavior() { 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); // Warmup should be considered complete when disabled @@ -786,7 +786,7 @@ public void testWarmupCompletesImmediatelyWhenLagIsZero() throws InterruptedExce 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(300000), 0) ); // Initially warmup is not complete @@ -826,7 +826,7 @@ public void testWarmupTimesOutAndProceeds() throws InterruptedException, Timeout 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(500), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(500), 0) ); warmupPoller.start(); @@ -855,7 +855,7 @@ public void testWarmupStateTransitions() throws InterruptedException { 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(300000), 0) ); // Initial state should be NONE @@ -874,52 +874,6 @@ public void testWarmupStateTransitions() throws InterruptedException { warmupPoller.close(); } - public void testIsWarmupFailOnTimeout() { - // Test with fail on timeout = true - DefaultStreamPoller failOnTimeoutPoller = new DefaultStreamPoller( - new FakeIngestionSource.FakeIngestionShardPointer(0), - fakeConsumerFactory, - "", - 0, - partitionedBlockingQueueContainer, - StreamPoller.ResetState.NONE, - "", - errorStrategy, - StreamPoller.State.NONE, - 1000, - 1000, - 10000, - indexSettings, - new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(300000), 0, true) - ); - - assertTrue(failOnTimeoutPoller.isWarmupFailOnTimeout()); - failOnTimeoutPoller.close(); - - // Test with fail on timeout = false - DefaultStreamPoller proceedOnTimeoutPoller = new DefaultStreamPoller( - new FakeIngestionSource.FakeIngestionShardPointer(0), - fakeConsumerFactory, - "", - 0, - partitionedBlockingQueueContainer, - StreamPoller.ResetState.NONE, - "", - errorStrategy, - StreamPoller.State.NONE, - 1000, - 1000, - 10000, - indexSettings, - new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(300000), 0, false) - ); - - assertFalse(proceedOnTimeoutPoller.isWarmupFailOnTimeout()); - proceedOnTimeoutPoller.close(); - } - public void testWarmupCompletesViaTimeoutWhenPointerLagNegative() throws InterruptedException, TimeoutException { // Create a mock consumer factory that returns negative pointer-based lag (unsupported) IngestionConsumerFactory mockFactory = mock(IngestionConsumerFactory.class); @@ -943,7 +897,7 @@ public void testWarmupCompletesViaTimeoutWhenPointerLagNegative() throws Interru 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(500), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(500), 0) ); warmupPoller.start(); @@ -979,7 +933,7 @@ public void testWarmupWithNonZeroLagThreshold() throws InterruptedException, Tim 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(true, TimeValue.timeValueMillis(30000), 100, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(30000), 100) ); warmupPoller.start(); @@ -1009,7 +963,7 @@ public void testWarmupAwaitReturnsImmediatelyWhenAlreadyComplete() throws Interr 10000, indexSettings, new DefaultIngestionMessageMapper(), - new IngestionSource.WarmupConfig(false, TimeValue.timeValueMillis(300000), 0, false) + new IngestionSource.WarmupConfig(TimeValue.timeValueMillis(-1), 0) ); // Should return immediately without blocking since warmup is disabled