diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 31bc65240d294..96acb3b58d099 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1641,6 +1641,20 @@
+
+ ozone.om.exception.stacktrace.propagate
+ true
+ OZONE, OM, DEBUG
+
+ If true, propagate full stacktrace for system exceptions to the client.
+ If false, propagate summary message only and log stacktrace on server.
+ Full stacktrace on the client is useful for developers to debug
+ unexpected system errors encountered on the server while handling a
+ request.
+ This setting is not applicable for business exceptions.
+
+
+
ozone.om.ratis.client.request.timeout.duration
3s
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
index dcb9b5cdeac95..100c955678247 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -151,6 +151,13 @@ private OMConfigKeys() {
public static final TimeDuration OZONE_OM_RATIS_MINIMUM_TIMEOUT_DEFAULT
= TimeDuration.valueOf(1, TimeUnit.SECONDS);
+ // Propagate stack trace for System Exceptions to the client.
+ public static final String OZONE_OM_PROPAGATE_SYSTEM_EXCEPTION_STACKTRACE
+ = "ozone.om.exception.stacktrace.propagate";
+ public static final boolean
+ OZONE_OM_PROPAGATE_SYSTEM_EXCEPTION_STACKTRACE_DEFAULT
+ = true;
+
// OM Ratis client configurations
public static final String OZONE_OM_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY
= "ozone.om.ratis.client.request.timeout.duration";
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index a6503d73140a3..76a940b8e16f3 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -200,15 +200,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_FILE;
import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_TEMP_FILE;
import static org.apache.hadoop.ozone.OzoneConsts.RPC_PORT;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_DEFAULT;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME;
-import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME_DEFAULT;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.*;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_AUTH_METHOD;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
@@ -1193,8 +1185,12 @@ private RPC.Server getRpcServer(OzoneConfiguration conf) throws IOException {
OZONE_OM_HANDLER_COUNT_DEFAULT);
RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class,
ProtobufRpcEngine.class);
+ final boolean propagateExceptionStack =
+ conf.getBoolean(OZONE_OM_PROPAGATE_SYSTEM_EXCEPTION_STACKTRACE,
+ OZONE_OM_PROPAGATE_SYSTEM_EXCEPTION_STACKTRACE_DEFAULT);
this.omServerProtocol = new OzoneManagerProtocolServerSideTranslatorPB(
- this, omRatisServer, omClientProtocolMetrics, isRatisEnabled);
+ this, omRatisServer, omClientProtocolMetrics, isRatisEnabled,
+ propagateExceptionStack);
BlockingService omService = newReflectiveBlockingService(omServerProtocol);
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
index d4c029b8b3b99..cccae421f5b08 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
@@ -22,6 +22,7 @@
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.NotLeaderException;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerDoubleBuffer;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
@@ -56,6 +57,7 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
private final OzoneManagerRatisServer omRatisServer;
private final RequestHandler handler;
private final boolean isRatisEnabled;
+ private final boolean propagateExceptionStack;
private final OzoneManager ozoneManager;
private final OzoneManagerDoubleBuffer ozoneManagerDoubleBuffer;
private final AtomicLong transactionIndex = new AtomicLong(0L);
@@ -71,11 +73,13 @@ public OzoneManagerProtocolServerSideTranslatorPB(
OzoneManager impl,
OzoneManagerRatisServer ratisServer,
ProtocolMessageMetrics metrics,
- boolean enableRatis) {
+ boolean enableRatis,
+ boolean propagateExceptionStack) {
this.ozoneManager = impl;
handler = new OzoneManagerRequestHandler(impl);
this.omRatisServer = ratisServer;
this.isRatisEnabled = enableRatis;
+ this.propagateExceptionStack = propagateExceptionStack;
this.ozoneManagerDoubleBuffer =
new OzoneManagerDoubleBuffer(ozoneManager.getMetadataManager(), (i) -> {
// Do nothing.
@@ -118,7 +122,7 @@ private OMResponse processRequest(OMRequest request) throws
request = omClientRequest.preExecute(ozoneManager);
} catch (IOException ex) {
// As some of the preExecute returns error. So handle here.
- return createErrorResponse(request, ex);
+ return failRequestWithException(request, ex);
}
return submitRequestToRatis(request);
} else {
@@ -131,7 +135,27 @@ private OMResponse processRequest(OMRequest request) throws
}
}
} else {
- return submitRequestDirectlyToOM(request);
+ try {
+ OMResponse response = submitRequestDirectlyToOM(request);
+ return response;
+ } catch (IOException ex) {
+ return failRequestWithException(request, ex);
+ }
+ }
+ }
+
+ private OMResponse failRequestWithException(OMRequest request, IOException e)
+ throws ServiceException {
+ // send summary error response for business exceptions.
+ if (e instanceof OMException) {
+ return createErrorResponse(request, e);
+ }
+ // propagate full stack trace for System Exceptions?
+ if (propagateExceptionStack) {
+ throw new ServiceException(e.getMessage(), e);
+ } else {
+ LOG.error(e.getMessage(), e);
+ return createErrorResponse(request, e);
}
}
@@ -202,7 +226,8 @@ private ServiceException createNotLeaderException() {
/**
* Submits request directly to OM.
*/
- private OMResponse submitRequestDirectlyToOM(OMRequest request) {
+ private OMResponse submitRequestDirectlyToOM(OMRequest request)
+ throws IOException {
OMClientResponse omClientResponse = null;
long index = 0L;
try {
@@ -219,7 +244,7 @@ private OMResponse submitRequestDirectlyToOM(OMRequest request) {
omClientResponse = omClientRequest.validateAndUpdateCache(
ozoneManager, index, ozoneManagerDoubleBuffer::add);
}
- } catch(IOException ex) {
+ } catch(OMException ex) {
// As some of the preExecute returns error. So handle here.
return createErrorResponse(request, ex);
}