Introduce limit to the number of terms in Terms Query (#27968)

- Introduce index level settings to control the maximum number of terms
    that can be used in a Terms Query
- Issue a deprecation warning if a request exceeds this max number

Closes #18829

Author: mayya-sharipova

core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -119,6 +119,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexSettings.MAX_RESCORE_WINDOW_SETTING,
         IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING,
         IndexSettings.MAX_ANALYZED_OFFSET_SETTING,
+        IndexSettings.MAX_TERMS_COUNT_SETTING,
         IndexSettings.INDEX_TRANSLOG_SYNC_INTERVAL_SETTING,
         IndexSettings.DEFAULT_FIELD_SETTING,
         IndexSettings.QUERY_STRING_LENIENT_SETTING,

core/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -132,6 +132,15 @@ public final class IndexSettings {
     public static final Setting<Integer> MAX_ANALYZED_OFFSET_SETTING =
         Setting.intSetting("index.highlight.max_analyzed_offset", 10000, 1, Property.Dynamic, Property.IndexScope);
 
+
+    /**
+     * Index setting describing the maximum number of terms that can be used in Terms Query.
+     * The default maximum of 65536 terms is defensive, as extra processing and memory is involved
+     * for each additional term, and a large number of terms degrade the cluster performance.
+     */
+    public static final Setting<Integer> MAX_TERMS_COUNT_SETTING =
+        Setting.intSetting("index.max_terms_count", 65536, 1, Property.Dynamic, Property.IndexScope);
+
     /**
      * Index setting describing for NGramTokenizer and NGramTokenFilter
      * the maximum difference between
@@ -288,6 +297,7 @@ public final class IndexSettings {
     private volatile int maxShingleDiff;
     private volatile boolean TTLPurgeDisabled;
     private volatile int maxAnalyzedOffset;
+    private volatile int maxTermsCount;
 
     /**
      * The maximum number of refresh listeners allows on this shard.
@@ -397,6 +407,7 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
         maxRefreshListeners = scopedSettings.get(MAX_REFRESH_LISTENERS_PER_SHARD);
         maxSlicesPerScroll = scopedSettings.get(MAX_SLICES_PER_SCROLL);
         maxAnalyzedOffset = scopedSettings.get(MAX_ANALYZED_OFFSET_SETTING);
+        maxTermsCount = scopedSettings.get(MAX_TERMS_COUNT_SETTING);
         this.mergePolicyConfig = new MergePolicyConfig(logger, this);
         this.indexSortConfig = new IndexSortConfig(this);
         singleType = INDEX_MAPPING_SINGLE_TYPE_SETTING.get(indexMetaData.getSettings()); // get this from metadata - it's not registered
@@ -438,6 +449,7 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
         scopedSettings.addSettingsUpdateConsumer(INDEX_REFRESH_INTERVAL_SETTING, this::setRefreshInterval);
         scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners);
         scopedSettings.addSettingsUpdateConsumer(MAX_ANALYZED_OFFSET_SETTING, this::setHighlightMaxAnalyzedOffset);
+        scopedSettings.addSettingsUpdateConsumer(MAX_TERMS_COUNT_SETTING, this::setMaxTermsCount);
         scopedSettings.addSettingsUpdateConsumer(MAX_SLICES_PER_SCROLL, this::setMaxSlicesPerScroll);
         scopedSettings.addSettingsUpdateConsumer(DEFAULT_FIELD_SETTING, this::setDefaultFields);
     }
@@ -717,6 +729,13 @@ private void setMaxDocvalueFields(int maxDocvalueFields) {
 
     private void setHighlightMaxAnalyzedOffset(int maxAnalyzedOffset) { this.maxAnalyzedOffset = maxAnalyzedOffset; }
 
+    /**
+     *  Returns the maximum number of terms that can be used in a Terms Query request
+     */
+    public int getMaxTermsCount() { return this.maxTermsCount; }
+
+    private void setMaxTermsCount (int maxTermsCount) { this.maxTermsCount = maxTermsCount; }
+
     /**
      * Returns the maximum number of allowed script_fields to retrieve in a search request
      */

core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java

@@ -34,11 +34,14 @@
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.indices.TermsLookup;
 
@@ -59,6 +62,7 @@
  * A filter for a field based on several terms matching on any of them.
  */
 public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(TermsQueryBuilder.class));
     public static final String NAME = "terms";
 
     private final String fieldName;
@@ -416,6 +420,13 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
         if (values == null || values.isEmpty()) {
             return Queries.newMatchNoDocsQuery("No terms supplied for \"" + getName() + "\" query.");
         }
+        int maxTermsCount = context.getIndexSettings().getMaxTermsCount();
+        if (values.size() > maxTermsCount){
+            DEPRECATION_LOGGER.deprecated(
+                "Deprecated: the number of terms ["  + values.size() +  "] used in the Terms Query request has exceeded " +
+                    "the allowed maximum of [" + maxTermsCount + "]. " + "This maximum can be set by changing the [" +
+                    IndexSettings.MAX_TERMS_COUNT_SETTING.getKey() + "] index level setting.");
+        }
         MappedFieldType fieldType = context.fieldMapper(fieldName);
 
         if (fieldType != null) {

docs/reference/index-modules.asciidoc

@@ -188,6 +188,11 @@ specific index module:
      This setting is only applicable when highlighting is requested on a text that was indexed without offsets or term vectors.
      Defaults to `10000`.
 
+ `index.max_terms_count`::
+
+    The maximum number of terms that can be used in Terms Query.
+    Defaults to `65536`.
+
 [float]
 === Settings in other index modules
 

docs/reference/migration/migrate_6_0/search.asciidoc

@@ -191,3 +191,11 @@ Setting `request_cache:true` on a query that creates a scroll ('scroll=1m`)
 In future versions we will return a `400 - Bad request` instead of just ignoring
 the hint.
 Scroll queries are not meant to be cached.
+
+==== Limiting the number of terms that can be used in a Terms Query request
+
+Executing a Terms Query with a lot of terms may degrade the cluster performance,
+as each additional term demands extra processing and memory.
+To safeguard against this, the maximum number of terms that can be used in a
+Terms Query request has been limited to 65536. This default maximum can be changed
+for a particular index with the index setting `index.max_terms_count`.

docs/reference/query-dsl/terms-query.asciidoc

@@ -57,6 +57,14 @@ across all nodes if the "reference" terms data is not large. The lookup
 terms filter will prefer to execute the get request on a local node if
 possible, reducing the need for networking.
 
+[WARNING]
+Executing a Terms Query request with a lot of terms can be quite slow,
+as each additional term demands extra processing and memory.
+To safeguard against this, the maximum number of terms that can be used
+in a Terms Query both directly or through lookup has been limited to `65536`.
+This default maximum can be changed for a particular index with the index setting
+ `index.max_terms_count`.
+
 [float]
 ===== Terms lookup twitter example
 At first we index the information for user with id 2, specifically, its

rest-api-spec/src/main/resources/rest-api-spec/test/search/170_terms_query.yml

@@ -0,0 +1,58 @@
+---
+"Terms Query with No.of terms exceeding index.max_terms_count should produce a warning":
+  - skip:
+      version: " - 6.1.99"
+      reason: index.max_terms_count setting has been added in 6.2
+      features: "warnings"
+  - do:
+      indices.create:
+          index: test_index
+          body:
+              settings:
+                  number_of_shards: 1
+                  index.max_terms_count: 2
+              mappings:
+                  test_type:
+                      properties:
+                          user:
+                              type: keyword
+                          followers:
+                              type: keyword
+  - do:
+      bulk:
+          refresh: true
+          body:
+              - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "u1"}}'
+              - '{"user": "u1", "followers": ["u2", "u3"]}'
+              - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "u2"}}'
+              - '{"user": "u2", "followers": ["u1", "u3", "u4"]}'
+              - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "u3"}}'
+              - '{"user": "u3", "followers": ["u1"]}'
+              - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "u4"}}'
+              - '{"user": "u4", "followers": ["u3"]}'
+
+  - do:
+      search:
+          index: test_index
+          body: {"query" : {"terms" : {"user" : ["u1", "u2"]}}}
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+          index: test_index
+          body: {"query" : {"terms" : {"user" : ["u1", "u2", "u3"]}}}
+      warnings:
+          - "Deprecated: the number of terms [3] used in the Terms Query request has exceeded the allowed maximum of [2]. This maximum can be set by changing the [index.max_terms_count] index level setting."
+
+  - do:
+      search:
+          index: test_index
+          body: {"query" : {"terms" : {"user" : {"index" : "test_index", "type" : "test_type", "id" : "u1", "path" : "followers"}}}}
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+          index: test_index
+          body: {"query" : {"terms" : {"user" : {"index" : "test_index", "type" : "test_type", "id" : "u2", "path" : "followers"}}}}
+      warnings:
+          - "Deprecated: the number of terms [3] used in the Terms Query request has exceeded the allowed maximum of [2]. This maximum can be set by changing the [index.max_terms_count] index level setting."