[Star tree] Remove star tree feature flag and add index setting to configure star tree search on index basis (opensearch-project#18070)

* Removing star tree indexing and queries added till 2.19 out of experimental

Signed-off-by: bharath-techie <[email protected]>

* Removing star tree feature flag

Signed-off-by: bharath-techie <[email protected]>

* Adding more tests for feature flag

Signed-off-by: bharath-techie <[email protected]>

* Adding Changelog

Signed-off-by: bharath-techie <[email protected]>

* addressing review comment

Signed-off-by: bharath-techie <[email protected]>

* resolving conflicts

Signed-off-by: bharath-techie <[email protected]>

---------

Signed-off-by: bharath-techie <[email protected]>Signed-off-by: TJ Neuenfeldt <[email protected]>

Author: bharath-techie

CHANGELOG.md

@@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add FIPS build tooling ([#4254](https://github.com/opensearch-project/security/issues/4254))
 - Support Nested Aggregations as part of Star-Tree ([#18048](https://github.com/opensearch-project/OpenSearch/pull/18048))
 - [Star-Tree] Support for date-range queries with star-tree supported aggregations ([#17855](https://github.com/opensearch-project/OpenSearch/pull/17855)
+- [Star tree] Remove star tree feature flag and add index setting to configure star tree search on index basis ([#18070](https://github.com/opensearch-project/OpenSearch/pull/18070))
 - Approximation Framework Enhancement: Update the BKD traversal logic to improve the performance on skewed data ([#18439](https://github.com/opensearch-project/OpenSearch/issues/18439))
 
 ### Changed

distribution/src/config/opensearch.yml

@@ -121,7 +121,3 @@ ${path.logs}
 # Once there is no observed impact on performance, this feature flag can be removed.
 #
 #opensearch.experimental.optimization.datetime_formatter_caching.enabled: false
-#
-# Gates the functionality of star tree index, which improves the performance of search aggregations.
-#
-#opensearch.experimental.feature.composite_index.star_tree.enabled: true

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -56,7 +56,6 @@
 import java.util.List;
 
 import static java.util.Collections.singletonList;
-import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX;
 import static org.hamcrest.Matchers.containsString;
 
 public class ScaledFloatFieldMapperTests extends MapperTestCase {
@@ -104,7 +103,6 @@ public void testExistsQueryDocValuesDisabled() throws IOException {
         assertParseMinimalWarnings();
     }
 
-    @LockFeatureFlag(STAR_TREE_INDEX)
     public void testScaledFloatWithStarTree() throws Exception {
 
         double scalingFactorField1 = randomDouble() * 100;

server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

@@ -15,7 +15,6 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.Rounding;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.index.Index;
@@ -581,11 +580,6 @@ private static String getMetric(boolean hasDocValues, boolean isKeyword) {
         return "numeric_dv";
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.STAR_TREE_INDEX, "true").build();
-    }
-
     @Before
     public final void setupNodeSettings() {
         Settings request = Settings.builder().put(CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey(), true).build();

server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java

@@ -35,7 +35,6 @@ protected FeatureFlagSettings(
         FeatureFlags.DATETIME_FORMATTER_CACHING_SETTING,
         FeatureFlags.WRITABLE_WARM_INDEX_SETTING,
         FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
-        FeatureFlags.STAR_TREE_INDEX_SETTING,
         FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING,
         FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE_SETTING,
         FeatureFlags.ARROW_STREAMS_SETTING,

server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java

@@ -260,6 +260,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING,
                 StarTreeIndexSettings.STAR_TREE_MAX_BASE_METRICS_SETTING,
                 StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING,
+                StarTreeIndexSettings.STAR_TREE_SEARCH_ENABLED_SETTING,
 
                 IndexSettings.INDEX_CONTEXT_CREATED_VERSION,
                 IndexSettings.INDEX_CONTEXT_CURRENT_VERSION,

server/src/main/java/org/opensearch/common/util/FeatureFlags.java

@@ -109,13 +109,6 @@ public class FeatureFlags {
         Property.NodeScope
     );
 
-    /**
-     * Gates the functionality of star tree index, which improves the performance of search
-     * aggregations.
-     */
-    public static final String STAR_TREE_INDEX = FEATURE_FLAG_PREFIX + "composite_index.star_tree.enabled";
-    public static final Setting<Boolean> STAR_TREE_INDEX_SETTING = Setting.boolSetting(STAR_TREE_INDEX, false, Property.NodeScope);
-
     /**
      * Gates the functionality of application based configuration templates.
      */
@@ -154,7 +147,6 @@ static class FeatureFlagsImpl {
                 put(TELEMETRY_SETTING, TELEMETRY_SETTING.getDefault(Settings.EMPTY));
                 put(DATETIME_FORMATTER_CACHING_SETTING, DATETIME_FORMATTER_CACHING_SETTING.getDefault(Settings.EMPTY));
                 put(WRITABLE_WARM_INDEX_SETTING, WRITABLE_WARM_INDEX_SETTING.getDefault(Settings.EMPTY));
-                put(STAR_TREE_INDEX_SETTING, STAR_TREE_INDEX_SETTING.getDefault(Settings.EMPTY));
                 put(
                     APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING,
                     APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING.getDefault(Settings.EMPTY)

server/src/main/java/org/opensearch/index/IndexSettings.java

@@ -930,8 +930,16 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
      */
     private volatile double docIdFuzzySetFalsePositiveProbability;
 
+    /**
+     * Denotes whether this is a composite index i.e star-tree index etc
+     */
     private final boolean isCompositeIndex;
 
+    /**
+     * Denotes whether search via star tree index is enabled for this index
+     */
+    private volatile boolean isStarTreeIndexEnabled;
+
     /**
      * Returns the default search fields for this index.
      */
@@ -1096,6 +1104,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         setEnableFuzzySetForDocId(scopedSettings.get(INDEX_DOC_ID_FUZZY_SET_ENABLED_SETTING));
         setDocIdFuzzySetFalsePositiveProbability(scopedSettings.get(INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING));
         isCompositeIndex = scopedSettings.get(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING);
+        isStarTreeIndexEnabled = scopedSettings.get(StarTreeIndexSettings.STAR_TREE_SEARCH_ENABLED_SETTING);
         scopedSettings.addSettingsUpdateConsumer(
             TieredMergePolicyProvider.INDEX_COMPOUND_FORMAT_SETTING,
             tieredMergePolicyProvider::setNoCFSRatio
@@ -1220,6 +1229,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
             IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING,
             this::setRemoteStoreTranslogRepository
         );
+        scopedSettings.addSettingsUpdateConsumer(StarTreeIndexSettings.STAR_TREE_SEARCH_ENABLED_SETTING, this::setStarTreeIndexEnabled);
     }
 
     private void setSearchIdleAfter(TimeValue searchIdleAfter) {
@@ -1790,6 +1800,14 @@ public long getGcDeletesInMillis() {
         return gcDeletesInMillis;
     }
 
+    public void setStarTreeIndexEnabled(boolean value) {
+        this.isStarTreeIndexEnabled = value;
+    }
+
+    public boolean getStarTreeIndexEnabled() {
+        return isStarTreeIndexEnabled;
+    }
+
     /**
      * Returns the merge policy that should be used for this index.
      *

server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java

@@ -12,7 +12,6 @@
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
 
@@ -25,16 +24,7 @@
 public class CompositeIndexSettings {
     public static final Setting<Boolean> STAR_TREE_INDEX_ENABLED_SETTING = Setting.boolSetting(
         "indices.composite_index.star_tree.enabled",
-        false,
-        value -> {
-            if (FeatureFlags.isEnabled(FeatureFlags.STAR_TREE_INDEX_SETTING) == false && value == true) {
-                throw new IllegalArgumentException(
-                    "star tree index is under an experimental feature and can be activated only by enabling "
-                        + FeatureFlags.STAR_TREE_INDEX_SETTING.getKey()
-                        + " feature flag in the JVM options"
-                );
-            }
-        },
+        true,
         Setting.Property.NodeScope,
         Setting.Property.Dynamic
     );

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeIndexSettings.java

@@ -134,4 +134,11 @@ public static DateTimeUnitRounding getTimeUnit(String expression) {
         Setting.Property.IndexScope,
         Setting.Property.Final
     );
+
+    public static final Setting<Boolean> STAR_TREE_SEARCH_ENABLED_SETTING = Setting.boolSetting(
+        "index.search.star_tree_index.enabled",
+        true,
+        Setting.Property.IndexScope,
+        Setting.Property.Dynamic
+    );
 }