Handle unmapped fields in _field_caps API

client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

@@ -1255,6 +1255,8 @@ public void testFieldCaps() throws IOException {
         FieldCapabilitiesResponse response = execute(request,
             highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync);
 
+        assertEquals(new String[] {"index1", "index2"}, response.getIndices());
+
         // Check the capabilities for the 'rating' field.
         assertTrue(response.get().containsKey("rating"));
         Map<String, FieldCapabilities> ratingResponse = response.getField("rating");

docs/reference/search/field-caps.asciidoc

@@ -71,6 +71,7 @@ GET _field_caps?fields=rating,title
 --------------------------------------------------
 {
     "fields": {
+        "indices": ["index1", "index2", "index3", "index4", "index5"],
         "rating": { <1>
             "long": {
                 "searchable": true,
@@ -103,9 +104,61 @@ and as a `keyword` in `index3` and `index4`.
 <3> The field `rating` is not searchable in `index4`.
 <4> The field `title` is defined as `text` in all indices.
 
+[float]
+=== Unmapped fields
 
+By default unmapped fields are ignored. You can include them in the response by
+adding a parameter called `include_unmapped` in the request:
 
+[source,js]
+--------------------------------------------------
+GET _field_caps?fields=rating,title&include_unmapped
+--------------------------------------------------
+// CONSOLE
 
+In which case the response will contain an entry for each field that is present in
+some indices but not all:
 
+[source,js]
+--------------------------------------------------
+{
+    "fields": {
+        "indices": ["index1", "index2", "index3"],
+        "rating": {
+            "long": {
+                "searchable": true,
+                "aggregatable": false,
+                "indices": ["index1", "index2"],
+                "non_aggregatable_indices": ["index1"]
+            },
+            "keyword": {
+                "searchable": false,
+                "aggregatable": true,
+                "indices": ["index3", "index4"],
+                "non_searchable_indices": ["index4"]
+            },
+            "unmapped": { <1>
+                "indices": ["index5"],
+                "searchable": false,
+                "aggregatable": false
+            }
+        },
+        "title": {
+            "text": {
+                "indices": ["index1", "index2", "index3", "index4"],
+                "searchable": true,
+                "aggregatable": false
+            },
+            "unmapped": { <2>
+                "indices": ["index5"]
+                "searchable": false,
+                "aggregatable": false
+            }
+        }
+    }
+}
+--------------------------------------------------
+// NOTCONSOLE
 
-
+<1> The `rating` field is unmapped` in `index5`.
+<2> The `title` field is unmapped` in `index5`.

rest-api-spec/src/main/resources/rest-api-spec/api/field_caps.json

@@ -32,6 +32,11 @@
           "options" : ["open","closed","none","all"],
           "default" : "open",
           "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
+        },
+        "include_unmapped": {
+          "type": "boolean",
+          "default": false,
+          "description": "Indicates whether unmapped fields should be included in the response."
         }
       }
     },

rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/10_basic.yml

@@ -14,8 +14,8 @@ setup:
                     type:     double
                   geo:
                     type:     geo_point
-                  date:
-                    type:     date
+                  misc:
+                    type:     text
                   object:
                     type: object
                     properties:
@@ -294,3 +294,28 @@ setup:
   - match: {fields.geo.keyword.indices:                 ["test3"]}
   - is_false: fields.geo.keyword.non_searchable_indices
   - is_false: fields.geo.keyword.on_aggregatable_indices
+
+---
+"Field caps with include_unmapped":
+  - skip:
+      version: " - 7.99.99"
+      reason: include_unmapped has been added in 7.1.0
+
+  - do:
+      field_caps:
+        include_unmapped: true
+        index: 'test1,test2,test3'
+        fields: [text, misc]
+
+  - match: {fields.text.text.searchable:                true}
+  - match: {fields.text.text.aggregatable:              false}
+  - is_false: fields.text.text.indices
+  - is_false: fields.text.text.non_searchable_indices
+  - is_false: fields.text.text.non_aggregatable_indices
+  - match: {fields.misc.text.searchable:                true}
+  - match: {fields.misc.text.aggregatable:              false}
+  - match: {fields.misc.text.indices:                   ["test1"]}
+  - match: {fields.misc.unmapped.searchable:            false}
+  - match: {fields.misc.unmapped.aggregatable:          false}
+  - match: {fields.misc.unmapped.indices:               ["test2", "test3"]}
+

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.action.fieldcaps;
 
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -34,6 +35,8 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * Describes the capabilities of a field optionally merged across multiple indices.
@@ -214,30 +217,30 @@ public String[] nonAggregatableIndices() {
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-
         FieldCapabilities that = (FieldCapabilities) o;
-
-        if (isSearchable != that.isSearchable) return false;
-        if (isAggregatable != that.isAggregatable) return false;
-        if (!name.equals(that.name)) return false;
-        if (!type.equals(that.type)) return false;
-        if (!Arrays.equals(indices, that.indices)) return false;
-        if (!Arrays.equals(nonSearchableIndices, that.nonSearchableIndices)) return false;
-        return Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices);
+        return isSearchable == that.isSearchable &&
+            isAggregatable == that.isAggregatable &&
+            Objects.equals(name, that.name) &&
+            Objects.equals(type, that.type) &&
+            Arrays.equals(indices, that.indices) &&
+            Arrays.equals(nonSearchableIndices, that.nonSearchableIndices) &&
+            Arrays.equals(nonAggregatableIndices, that.nonAggregatableIndices);
     }
 
     @Override
     public int hashCode() {
-        int result = name.hashCode();
-        result = 31 * result + type.hashCode();
-        result = 31 * result + (isSearchable ? 1 : 0);
-        result = 31 * result + (isAggregatable ? 1 : 0);
+        int result = Objects.hash(name, type, isSearchable, isAggregatable);
         result = 31 * result + Arrays.hashCode(indices);
         result = 31 * result + Arrays.hashCode(nonSearchableIndices);
         result = 31 * result + Arrays.hashCode(nonAggregatableIndices);
         return result;
     }
 
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
     static class Builder {
         private String name;
         private String type;
@@ -260,6 +263,10 @@ void add(String index, boolean search, boolean agg) {
             this.isAggregatable &= agg;
         }
 
+        List<String> getIndices() {
+            return indiceList.stream().map(c -> c.name).collect(Collectors.toList());
+        }
+
         FieldCapabilities build(boolean withIndices) {
             final String[] indices;
             /* Eclipse can't deal with o -> o.name, maybe because of

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java

@@ -28,8 +28,7 @@
 
 import java.io.IOException;
 
-public class FieldCapabilitiesIndexRequest
-    extends SingleShardRequest<FieldCapabilitiesIndexRequest> {
+public class FieldCapabilitiesIndexRequest extends SingleShardRequest<FieldCapabilitiesIndexRequest> {
 
     private String[] fields;
     private OriginalIndices originalIndices;

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java

@@ -26,6 +26,7 @@
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  * Response for {@link FieldCapabilitiesIndexRequest} requests.
@@ -89,14 +90,13 @@ public void writeTo(StreamOutput out) throws IOException {
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-
         FieldCapabilitiesIndexResponse that = (FieldCapabilitiesIndexResponse) o;
-
-        return responseMap.equals(that.responseMap);
+        return Objects.equals(indexName, that.indexName) &&
+            Objects.equals(responseMap, that.responseMap);
     }
 
     @Override
     public int hashCode() {
-        return responseMap.hashCode();
+        return Objects.hash(indexName, responseMap);
     }
 }

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.fieldcaps;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
@@ -44,15 +45,15 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
     private String[] indices = Strings.EMPTY_ARRAY;
     private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
     private String[] fields = Strings.EMPTY_ARRAY;
+    private boolean includeUnmapped = false;
     // pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
     private boolean mergeResults = true;
 
     private static ObjectParser<FieldCapabilitiesRequest, Void> PARSER =
         new ObjectParser<>(NAME, FieldCapabilitiesRequest::new);
 
     static {
-        PARSER.declareStringArray(fromList(String.class, FieldCapabilitiesRequest::fields),
-            FIELDS_FIELD);
+        PARSER.declareStringArray(fromList(String.class, FieldCapabilitiesRequest::fields), FIELDS_FIELD);
     }
 
     public FieldCapabilitiesRequest() {}
@@ -83,6 +84,11 @@ public void readFrom(StreamInput in) throws IOException {
         indices = in.readStringArray();
         indicesOptions = IndicesOptions.readIndicesOptions(in);
         mergeResults = in.readBoolean();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            includeUnmapped = in.readBoolean();
+        } else {
+            includeUnmapped = false;
+        }
     }
 
     @Override
@@ -92,6 +98,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeStringArray(indices);
         indicesOptions.writeIndicesOptions(out);
         out.writeBoolean(mergeResults);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeBoolean(includeUnmapped);
+        }
     }
 
     /**
@@ -123,6 +132,11 @@ public FieldCapabilitiesRequest indicesOptions(IndicesOptions indicesOptions) {
         return this;
     }
 
+    public FieldCapabilitiesRequest includeUnmapped(boolean includeUnmapped) {
+        this.includeUnmapped = includeUnmapped;
+        return this;
+    }
+
     @Override
     public String[] indices() {
         return indices;
@@ -133,12 +147,15 @@ public IndicesOptions indicesOptions() {
         return indicesOptions;
     }
 
+    public boolean includeUnmapped() {
+        return includeUnmapped;
+    }
+
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = null;
         if (fields == null || fields.length == 0) {
-            validationException =
-                ValidateActions.addValidationError("no fields specified", validationException);
+            validationException = ValidateActions.addValidationError("no fields specified", validationException);
         }
         return validationException;
     }
@@ -152,14 +169,12 @@ public boolean equals(Object o) {
         return  Arrays.equals(indices, that.indices) &&
             Objects.equals(indicesOptions, that.indicesOptions) &&
             Arrays.equals(fields, that.fields) &&
-            Objects.equals(mergeResults, that.mergeResults);
+            Objects.equals(mergeResults, that.mergeResults) &&
+            includeUnmapped == that.includeUnmapped;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(Arrays.hashCode(indices),
-            indicesOptions,
-            Arrays.hashCode(fields),
-            mergeResults);
+        return Objects.hash(Arrays.hashCode(indices), indicesOptions, Arrays.hashCode(fields), mergeResults, includeUnmapped);
     }
 }

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java

@@ -36,4 +36,9 @@ public FieldCapabilitiesRequestBuilder setFields(String... fields) {
         request().fields(fields);
         return this;
     }
+
+    public FieldCapabilitiesRequestBuilder setIncludeUnmapped(boolean includeUnmapped) {
+        request().includeUnmapped(includeUnmapped);
+        return this;
+    }
 }