diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java index b34ef863d565..498ab19b50e0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRequestFutureImpl.java @@ -219,13 +219,14 @@ public void run() { } catch (IOException e) { // The service itself failed . It may be an error coming from the communication // layer, but, as well, a functional error raised by the server. - receiveGlobalFailure(multiAction, server, numAttempt, e, true); + + receiveGlobalFailure(multiAction, server, numAttempt, e); return; } catch (Throwable t) { // This should not happen. Let's log & retry anyway. LOG.error("id=" + asyncProcess.id + ", caught throwable. Unexpected." + " Retrying. Server=" + server + ", tableName=" + tableName, t); - receiveGlobalFailure(multiAction, server, numAttempt, t, true); + receiveGlobalFailure(multiAction, server, numAttempt, t); return; } if (res.type() == AbstractResponse.ResponseType.MULTI) { @@ -570,7 +571,6 @@ private RegionLocations findAllLocationsOrFail(Action action, boolean useCache) */ void sendMultiAction(Map actionsByServer, int numAttempt, List actionsForReplicaThread, boolean reuseThread) { - boolean clearServerCache = true; // Run the last item on the same thread if we are already on a send thread. // We hope most of the time it will be the only item, so we can cut down on threads. int actionsRemaining = actionsByServer.size(); @@ -606,7 +606,6 @@ void sendMultiAction(Map actionsByServer, int numAttemp LOG.warn("id=" + asyncProcess.id + ", task rejected by pool. Unexpected." + " Server=" + server.getServerName(), t); // Do not update cache if exception is from failing to submit action to thread pool - clearServerCache = false; } else { // see #HBASE-14359 for more details LOG.warn("Caught unexpected exception/error: ", t); @@ -614,7 +613,7 @@ void sendMultiAction(Map actionsByServer, int numAttemp asyncProcess.decTaskCounters(multiAction.getRegions(), server); // We're likely to fail again, but this will increment the attempt counter, // so it will finish. - receiveGlobalFailure(multiAction, server, numAttempt, t, clearServerCache); + receiveGlobalFailure(multiAction, server, numAttempt, t); } } } @@ -764,13 +763,24 @@ private void failAll(MultiAction actions, ServerName server, int numAttempt, * @param t the throwable (if any) that caused the resubmit */ private void receiveGlobalFailure(MultiAction rsActions, ServerName server, int numAttempt, - Throwable t, boolean clearServerCache) { + Throwable t) { errorsByServer.reportServerError(server); Retry canRetry = errorsByServer.canTryMore(numAttempt) ? Retry.YES : Retry.NO_RETRIES_EXHAUSTED; + boolean clearServerCache; + + if (t instanceof RejectedExecutionException) { + clearServerCache = false; + } else { + clearServerCache = ClientExceptionsUtil.isMetaClearingException(t); + } // Do not update cache if exception is from failing to submit action to thread pool if (clearServerCache) { cleanServerCache(server, t); + + if (LOG.isTraceEnabled()) { + LOG.trace("Cleared meta cache for server {} due to global failure {}", server, t); + } } int failed = 0; @@ -779,12 +789,8 @@ private void receiveGlobalFailure(MultiAction rsActions, ServerName server, int for (Map.Entry> e : rsActions.actions.entrySet()) { byte[] regionName = e.getKey(); byte[] row = e.getValue().get(0).getAction().getRow(); - // Do not use the exception for updating cache because it might be coming from - // any of the regions in the MultiAction and do not update cache if exception is - // from failing to submit action to thread pool if (clearServerCache) { - updateCachedLocations(server, regionName, row, - ClientExceptionsUtil.isMetaClearingException(t) ? null : t); + updateCachedLocations(server, regionName, row, t); } for (Action action : e.getValue()) { Retry retry =