Search - allow_partial_search_results flag to search requests with default setting = true. (#27906)

When false will error if search either timeouts, has partial errors or has missing shards rather
than returning partial search results. Backport of 28440.

Closes #27435

Author: markharwood

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -444,6 +444,9 @@ static Request search(SearchRequest searchRequest) throws IOException {
         if (searchRequest.requestCache() != null) {
             params.putParam("request_cache", Boolean.toString(searchRequest.requestCache()));
         }
+        if (searchRequest.allowPartialSearchResults() != null) {
+            params.putParam("allow_partial_search_results", Boolean.toString(searchRequest.allowPartialSearchResults()));
+        }
         params.putParam("batched_reduce_size", Integer.toString(searchRequest.getBatchedReduceSize()));
         if (searchRequest.scroll() != null) {
             params.putParam("scroll", searchRequest.scroll().keepAlive());

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java

@@ -846,6 +846,10 @@ public void testSearch() throws Exception {
             searchRequest.requestCache(randomBoolean());
             expectedParams.put("request_cache", Boolean.toString(searchRequest.requestCache()));
         }
+        if (randomBoolean()) {
+            searchRequest.allowPartialSearchResults(randomBoolean());
+            expectedParams.put("allow_partial_search_results", Boolean.toString(searchRequest.allowPartialSearchResults()));
+        }
         if (randomBoolean()) {
             searchRequest.setBatchedReduceSize(randomIntBetween(2, Integer.MAX_VALUE));
         }

docs/reference/search/request-body.asciidoc

@@ -86,6 +86,12 @@ And here is a sample response:
     aggregations and suggestions (no top hits returned).
     See <<shard-request-cache>>.
 
+`allow_partial_search_results`::
+
+    Set to `false` to return an overall failure if the request would produce partial 
+    results. Defaults to true, which will allow partial results in the case of timeouts
+    or partial failures.
+
 `terminate_after`::
 
     The maximum number of documents to collect for each shard,
@@ -103,9 +109,9 @@ And here is a sample response:
 
 
 
-Out of the above, the `search_type` and the `request_cache` must be passed as
-query-string parameters. The rest of the search request should be passed
-within the body itself. The body content can also be passed as a REST
+Out of the above, the `search_type`, `request_cache` and the `allow_partial_search_results` 
+settings must be passed as query-string parameters. The rest of the search request should 
+be passed within the body itself. The body content can also be passed as a REST
 parameter named `source`.
 
 Both HTTP GET and HTTP POST can be used to execute search with body. Since not

docs/reference/search/uri-request.asciidoc

@@ -122,4 +122,8 @@ Defaults to no terminate_after.
 Defaults to `query_then_fetch`. See
 <<search-request-search-type,_Search Type_>> for
 more details on the different types of search that can be performed.
+
+|`allow_partial_search_results` |Set to `false` to return an overall failure if the request would produce
+partial results. Defaults to true, which will allow partial results in the case of timeouts
+or partial failures..
 |=======================================================================

modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollParallelizationHelper.java

@@ -164,6 +164,9 @@ static SearchRequest[] sliceIntoSubRequests(SearchRequest request, String field,
                     .requestCache(request.requestCache())
                     .scroll(request.scroll())
                     .indicesOptions(request.indicesOptions());
+            if (request.allowPartialSearchResults() != null) {
+                slices[slice].allowPartialSearchResults(request.allowPartialSearchResults());
+            }
         }
         return slices;
     }

rest-api-spec/src/main/resources/rest-api-spec/api/search.json

@@ -147,6 +147,11 @@
           "type" : "boolean",
           "description": "Indicate if the number of documents that match the query should be tracked"
         },
+        "allow_partial_search_results": {
+          "type" : "boolean",
+          "default" : true,
+          "description": "Indicate if an error should be returned if there is a partial search failure or timeout"
+        },
         "typed_keys": {
           "type" : "boolean",
           "description" : "Specify whether aggregation and suggester names should be prefixed by their respective types in the response"

server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java

@@ -131,13 +131,27 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha
             }
             onPhaseFailure(currentPhase, "all shards failed", cause);
         } else {
-            if (logger.isTraceEnabled()) {
-                final String resultsFrom = results.getSuccessfulResults()
-                    .map(r -> r.getSearchShardTarget().toString()).collect(Collectors.joining(","));
-                logger.trace("[{}] Moving to next phase: [{}], based on results from: {} (cluster state version: {})",
-                    currentPhase.getName(), nextPhase.getName(), resultsFrom, clusterStateVersion);
+            Boolean allowPartialResults = request.allowPartialSearchResults();
+            assert allowPartialResults != null : "SearchRequest missing setting for allowPartialSearchResults";            
+            if (allowPartialResults == false && shardFailures.get() != null ){
+                if (logger.isDebugEnabled()) {
+                    final ShardOperationFailedException[] shardSearchFailures = ExceptionsHelper.groupBy(buildShardFailures());
+                    Throwable cause = shardSearchFailures.length == 0 ? null :
+                        ElasticsearchException.guessRootCauses(shardSearchFailures[0].getCause())[0];
+                    logger.debug((Supplier<?>) () -> new ParameterizedMessage("{} shards failed for phase: [{}]", 
+                            shardSearchFailures.length, getName()),
+                        cause);
+                }
+                onPhaseFailure(currentPhase, "Partial shards failure", null);                
+            } else { 
+                if (logger.isTraceEnabled()) {
+                    final String resultsFrom = results.getSuccessfulResults()
+                        .map(r -> r.getSearchShardTarget().toString()).collect(Collectors.joining(","));
+                    logger.trace("[{}] Moving to next phase: [{}], based on results from: {} (cluster state version: {})",
+                        currentPhase.getName(), nextPhase.getName(), resultsFrom, clusterStateVersion);
+                }
+                executePhase(nextPhase);
             }
-            executePhase(nextPhase);
         }
     }
 
@@ -265,8 +279,16 @@ public final SearchRequest getRequest() {
 
     @Override
     public final SearchResponse buildSearchResponse(InternalSearchResponse internalSearchResponse, String scrollId) {
+        
+        ShardSearchFailure[] failures = buildShardFailures();
+        Boolean allowPartialResults = request.allowPartialSearchResults();
+        assert allowPartialResults != null : "SearchRequest missing setting for allowPartialSearchResults";
+        if (allowPartialResults == false && failures.length > 0){
+            raisePhaseFailure(new SearchPhaseExecutionException("", "Shard failures", null, failures));                        
+        }                              
+        
         return new SearchResponse(internalSearchResponse, scrollId, getNumShards(), successfulOps.get(),
-            skippedOps.get(), buildTookInMillis(), buildShardFailures(), clusters);
+            skippedOps.get(), buildTookInMillis(), failures, clusters);
     }
 
     @Override

server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java

@@ -128,6 +128,9 @@ private SearchRequest buildExpandSearchRequest(SearchRequest orig, SearchSourceB
             .preference(orig.preference())
             .routing(orig.routing())
             .searchType(orig.searchType());
+        if (orig.allowPartialSearchResults() != null){
+            groupRequest.allowPartialSearchResults(orig.allowPartialSearchResults());
+        }
         if (orig.isMaxConcurrentShardRequestsSet()) {
             groupRequest.setMaxConcurrentShardRequests(orig.getMaxConcurrentShardRequests());
         }

server/src/main/java/org/elasticsearch/action/search/InitialSearchPhase.java

@@ -146,7 +146,27 @@ public final void run() throws IOException {
         if (shardsIts.size() > 0) {
             int maxConcurrentShardRequests = Math.min(this.maxConcurrentShardRequests, shardsIts.size());
             final boolean success = shardExecutionIndex.compareAndSet(0, maxConcurrentShardRequests);
-            assert success;
+            assert success;            
+            assert request.allowPartialSearchResults() != null : "SearchRequest missing setting for allowPartialSearchResults";
+            if (request.allowPartialSearchResults() == false) {
+                final StringBuilder missingShards = new StringBuilder();
+                // Fail-fast verification of all shards being available
+                for (int index = 0; index < shardsIts.size(); index++) {
+                    final SearchShardIterator shardRoutings = shardsIts.get(index);
+                    if (shardRoutings.size() == 0) {
+                        if(missingShards.length() >0 ){
+                            missingShards.append(", ");                            
+                        }
+                        missingShards.append(shardRoutings.shardId());
+                    }
+                }
+                if (missingShards.length() >0) {
+                    //Status red - shard is missing all copies and would produce partial results for an index search
+                    final String msg = "Search rejected due to missing shards ["+ missingShards +
+                            "]. Consider using `allow_partial_search_results` setting to bypass this error.";
+                    throw new SearchPhaseExecutionException(getName(), msg, null, ShardSearchFailure.EMPTY_ARRAY);
+                }
+            }
             for (int index = 0; index < maxConcurrentShardRequests; index++) {
                 final SearchShardIterator shardRoutings = shardsIts.get(index);
                 assert shardRoutings.skip() == false;

server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java

@@ -226,6 +226,8 @@ public static void readMultiLineFormat(BytesReference data,
                             searchRequest.preference(nodeStringValue(value, null));
                         } else if ("routing".equals(entry.getKey())) {
                             searchRequest.routing(nodeStringValue(value, null));
+                        } else if ("allow_partial_search_results".equals(entry.getKey())) {
+                            searchRequest.allowPartialSearchResults(nodeBooleanValue(value, null));
                         }
                     }
                     defaultOptions = IndicesOptions.fromMap(source, defaultOptions);
@@ -297,6 +299,9 @@ public static byte[] writeMultiLineFormat(MultiSearchRequest multiSearchRequest,
                 if (request.routing() != null) {
                     xContentBuilder.field("routing", request.routing());
                 }
+                if (request.allowPartialSearchResults() != null) {
+                    xContentBuilder.field("allow_partial_search_results", request.allowPartialSearchResults());
+                }
                 xContentBuilder.endObject();
                 xContentBuilder.bytes().writeTo(output);
             }