Skip to content

HDDS-2206. Separate handling for OMException and IOException in the Ozone Manager. Contributed by Supratim Deka #1637

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
supratimdeka wants to merge 1 commit into from
Closed

HDDS-2206. Separate handling for OMException and IOException in the Ozone Manager. Contributed by Supratim Deka #1637

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions hadoop-hdds/common/src/main/resources/ozone-default.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1641,6 +1641,20 @@
</description>
</property>

<property>
<name>ozone.om.exception.stacktrace.propagate</name>
<value>true</value>
<tag>OZONE, OM, DEBUG</tag>
<description>
If true, propagate full stacktrace for system exceptions to the client.
If false, propagate summary message only and log stacktrace on server.
Full stacktrace on the client is useful for developers to debug
unexpected system errors encountered on the server while handling a
request.
This setting is not applicable for business exceptions.
</description>
</property>

<property>
<name>ozone.om.ratis.client.request.timeout.duration</name>
<value>3s</value>
Expand Down
7 changes: 7 additions & 0 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,13 @@ private OMConfigKeys() {
public static final TimeDuration OZONE_OM_RATIS_MINIMUM_TIMEOUT_DEFAULT
= TimeDuration.valueOf(1, TimeUnit.SECONDS);

// Propagate stack trace for System Exceptions to the client.
public static final String OZONE_OM_PROPAGATE_SYSTEM_EXCEPTION_STACKTRACE
= "ozone.om.exception.stacktrace.propagate";
public static final boolean
OZONE_OM_PROPAGATE_SYSTEM_EXCEPTION_STACKTRACE_DEFAULT
= true;

// OM Ratis client configurations
public static final String OZONE_OM_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY
= "ozone.om.ratis.client.request.timeout.duration";
Expand Down
16 changes: 6 additions & 10 deletions hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,15 +200,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_FILE;
import static org.apache.hadoop.ozone.OzoneConsts.OM_METRICS_TEMP_FILE;
import static org.apache.hadoop.ozone.OzoneConsts.RPC_PORT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.*;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_AUTH_METHOD;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
Expand Down Expand Up @@ -1193,8 +1185,12 @@ private RPC.Server getRpcServer(OzoneConfiguration conf) throws IOException {
OZONE_OM_HANDLER_COUNT_DEFAULT);
RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class,
ProtobufRpcEngine.class);
final boolean propagateExceptionStack =
conf.getBoolean(OZONE_OM_PROPAGATE_SYSTEM_EXCEPTION_STACKTRACE,
OZONE_OM_PROPAGATE_SYSTEM_EXCEPTION_STACKTRACE_DEFAULT);
this.omServerProtocol = new OzoneManagerProtocolServerSideTranslatorPB(
this, omRatisServer, omClientProtocolMetrics, isRatisEnabled);
this, omRatisServer, omClientProtocolMetrics, isRatisEnabled,
propagateExceptionStack);

BlockingService omService = newReflectiveBlockingService(omServerProtocol);

Expand Down
35 changes: 30 additions & 5 deletions ...n/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.NotLeaderException;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerDoubleBuffer;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
Expand Down Expand Up @@ -56,6 +57,7 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
private final OzoneManagerRatisServer omRatisServer;
private final RequestHandler handler;
private final boolean isRatisEnabled;
private final boolean propagateExceptionStack;
private final OzoneManager ozoneManager;
private final OzoneManagerDoubleBuffer ozoneManagerDoubleBuffer;
private final AtomicLong transactionIndex = new AtomicLong(0L);
Expand All @@ -71,11 +73,13 @@ public OzoneManagerProtocolServerSideTranslatorPB(
OzoneManager impl,
OzoneManagerRatisServer ratisServer,
ProtocolMessageMetrics metrics,
boolean enableRatis) {
boolean enableRatis,
boolean propagateExceptionStack) {
this.ozoneManager = impl;
handler = new OzoneManagerRequestHandler(impl);
this.omRatisServer = ratisServer;
this.isRatisEnabled = enableRatis;
this.propagateExceptionStack = propagateExceptionStack;
this.ozoneManagerDoubleBuffer =
new OzoneManagerDoubleBuffer(ozoneManager.getMetadataManager(), (i) -> {
// Do nothing.
Expand Down Expand Up @@ -118,7 +122,7 @@ private OMResponse processRequest(OMRequest request) throws
request = omClientRequest.preExecute(ozoneManager);
} catch (IOException ex) {
// As some of the preExecute returns error. So handle here.
return createErrorResponse(request, ex);
return failRequestWithException(request, ex);
}
return submitRequestToRatis(request);
} else {
Expand All @@ -131,7 +135,27 @@ private OMResponse processRequest(OMRequest request) throws
}
}
} else {
return submitRequestDirectlyToOM(request);
try {
OMResponse response = submitRequestDirectlyToOM(request);
return response;
} catch (IOException ex) {
return failRequestWithException(request, ex);
}
}
}

private OMResponse failRequestWithException(OMRequest request, IOException e)
throws ServiceException {
// send summary error response for business exceptions.
if (e instanceof OMException) {
return createErrorResponse(request, e);
}
// propagate full stack trace for System Exceptions?
if (propagateExceptionStack) {
throw new ServiceException(e.getMessage(), e);
} else {
LOG.error(e.getMessage(), e);
return createErrorResponse(request, e);
}
}

Expand Down Expand Up @@ -202,7 +226,8 @@ private ServiceException createNotLeaderException() {
/**
* Submits request directly to OM.
*/
private OMResponse submitRequestDirectlyToOM(OMRequest request) {
private OMResponse submitRequestDirectlyToOM(OMRequest request)
throws IOException {
OMClientResponse omClientResponse = null;
long index = 0L;
try {
Expand All @@ -219,7 +244,7 @@ private OMResponse submitRequestDirectlyToOM(OMRequest request) {
omClientResponse = omClientRequest.validateAndUpdateCache(
ozoneManager, index, ozoneManagerDoubleBuffer::add);
}
} catch(IOException ex) {
} catch(OMException ex) {
// As some of the preExecute returns error. So handle here.
return createErrorResponse(request, ex);
}
Expand Down