diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 0e27ac05653b..41b212e7aeae 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -277,6 +277,7 @@ import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse.PrepareStatus; import org.apache.ratis.proto.RaftProtos.RaftPeerRole; import org.apache.ratis.protocol.RaftGroupId; +import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.FileUtils; @@ -931,6 +932,7 @@ private OzoneDelegationTokenSecretManager createDelegationTokenSecretManager( .setTokenRenewInterval(tokenRenewInterval) .setTokenRemoverScanInterval(tokenRemoverScanInterval) .setService(omRpcAddressTxt) + .setOzoneManager(this) .setS3SecretManager(s3SecretManager) .setCertificateClient(certClient) .setOmServiceId(omNodeDetails.getServiceId()) @@ -1095,7 +1097,7 @@ private RPC.Server getRpcServer(OzoneConfiguration conf) throws IOException { OzoneManagerService.newReflectiveBlockingService(omServerProtocol); OMInterServiceProtocolServerSideImpl omInterServerProtocol = - new OMInterServiceProtocolServerSideImpl(omRatisServer, + new OMInterServiceProtocolServerSideImpl(this, omRatisServer, isRatisEnabled); BlockingService omInterService = OzoneManagerInterService.newReflectiveBlockingService( @@ -3919,6 +3921,34 @@ public boolean isLeaderReady() { omRatisServer.checkLeaderStatus() == LEADER_AND_READY : true; } + /** + * Check the leader status. + * + * @return null leader is ready + * OMLeaderNotReadyException leader is not ready + * OMNotLeaderException not leader + */ + public void checkLeaderStatus() throws OMNotLeaderException, + OMLeaderNotReadyException { + OzoneManagerRatisServer.RaftServerStatus raftServerStatus = + omRatisServer.checkLeaderStatus(); + RaftPeerId raftPeerId = omRatisServer.getRaftPeerId(); + + switch (raftServerStatus) { + case LEADER_AND_READY: return; + case LEADER_AND_NOT_READY: + throw new OMLeaderNotReadyException( + raftPeerId.toString() + " is Leader " + + "but not ready to process request yet."); + case NOT_LEADER: + // TODO: Set suggest leaderID. Right now, client is not using suggest + // leaderID. Need to fix this. + throw new OMNotLeaderException(raftPeerId); + default: throw new IllegalStateException( + "Unknown Ratis Server state: " + raftServerStatus); + } + } + /** * Return if Ratis is enabled or not. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 86254064c367..3c1c59644bd7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -35,7 +35,6 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.exceptions.OMLeaderNotReadyException; import org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException; -import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus; import org.apache.hadoop.ozone.om.request.BucketLayoutAwareOMKeyRequestFactory; import org.apache.hadoop.ozone.om.request.bucket.OMBucketCreateRequest; import org.apache.hadoop.ozone.om.request.bucket.OMBucketDeleteRequest; @@ -85,7 +84,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; -import org.apache.ratis.protocol.RaftPeerId; import org.rocksdb.RocksDBException; import java.io.IOException; @@ -431,43 +429,13 @@ public static String getOMRatisSnapshotDirectory(ConfigurationSource conf) { return snapshotDir; } - public static void checkLeaderStatus(RaftServerStatus raftServerStatus, - RaftPeerId raftPeerId) throws ServiceException { - switch (raftServerStatus) { - case LEADER_AND_READY: return; - - case LEADER_AND_NOT_READY: throw createLeaderNotReadyException(raftPeerId); - - case NOT_LEADER: throw createNotLeaderException(raftPeerId); - - default: throw new IllegalStateException( - "Unknown Ratis Server state: " + raftServerStatus); + public static void checkLeaderStatus(OzoneManager ozoneManager) + throws ServiceException { + try { + ozoneManager.checkLeaderStatus(); + } catch (OMNotLeaderException | OMLeaderNotReadyException e) { + LOG.debug(e.getMessage()); + throw new ServiceException(e); } } - - private static ServiceException createNotLeaderException( - RaftPeerId raftPeerId) { - - // TODO: Set suggest leaderID. Right now, client is not using suggest - // leaderID. Need to fix this. - - OMNotLeaderException notLeaderException = - new OMNotLeaderException(raftPeerId); - - LOG.debug(notLeaderException.getMessage()); - - return new ServiceException(notLeaderException); - } - - private static ServiceException createLeaderNotReadyException( - RaftPeerId raftPeerId) { - - OMLeaderNotReadyException leaderNotReadyException = - new OMLeaderNotReadyException(raftPeerId.toString() + " is Leader " + - "but not ready to process request yet."); - - LOG.debug(leaderNotReadyException.getMessage()); - - return new ServiceException(leaderNotReadyException); - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java index f419301d6099..1e9786abf931 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMAdminProtocolServerSideImpl.java @@ -84,7 +84,7 @@ public DecommissionOMResponse decommission(RpcController controller, } OzoneManagerRatisServer omRatisServer = ozoneManager.getOmRatisServer(); - checkLeaderStatus(omRatisServer); + OzoneManagerRatisUtils.checkLeaderStatus(ozoneManager); OMNodeDetails decommNode = ozoneManager.getPeerNode(request.getNodeId()); if (decommNode == null) { @@ -107,10 +107,4 @@ public DecommissionOMResponse decommission(RpcController controller, .setSuccess(true) .build(); } - - private void checkLeaderStatus(OzoneManagerRatisServer omRatisServer) - throws ServiceException { - OzoneManagerRatisUtils.checkLeaderStatus(omRatisServer.checkLeaderStatus(), - omRatisServer.getRaftPeerId()); - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMInterServiceProtocolServerSideImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMInterServiceProtocolServerSideImpl.java index d5ceb4e8b853..48bc12c3c0e8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMInterServiceProtocolServerSideImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OMInterServiceProtocolServerSideImpl.java @@ -20,6 +20,8 @@ import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; import java.io.IOException; + +import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OMNodeDetails; import org.apache.hadoop.ozone.om.protocolPB.OMInterServiceProtocolPB; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; @@ -38,9 +40,11 @@ public class OMInterServiceProtocolServerSideImpl implements private final OzoneManagerRatisServer omRatisServer; private final boolean isRatisEnabled; + private final OzoneManager ozoneManager; - public OMInterServiceProtocolServerSideImpl( + public OMInterServiceProtocolServerSideImpl(OzoneManager ozoneMgr, OzoneManagerRatisServer ratisServer, boolean enableRatis) { + this.ozoneManager = ozoneMgr; this.omRatisServer = ratisServer; this.isRatisEnabled = enableRatis; } @@ -60,7 +64,7 @@ public BootstrapOMResponse bootstrap(RpcController controller, .build(); } - checkLeaderStatus(); + OzoneManagerRatisUtils.checkLeaderStatus(ozoneManager); OMNodeDetails newOmNode = new OMNodeDetails.Builder() .setOMNodeId(request.getNodeId()) @@ -82,9 +86,4 @@ public BootstrapOMResponse bootstrap(RpcController controller, .setSuccess(true) .build(); } - - private void checkLeaderStatus() throws ServiceException { - OzoneManagerRatisUtils.checkLeaderStatus(omRatisServer.checkLeaderStatus(), - omRatisServer.getRaftPeerId()); - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java index ff3884cb9ddd..8e1763562778 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java @@ -160,7 +160,6 @@ private OMResponse processRequest(OMRequest request) throws // if current OM is leader and then proceed with processing the request. if (request.hasS3Authentication()) { s3Auth = true; - checkLeaderStatus(); S3SecurityUtil.validateS3Credential(request, ozoneManager); } } catch (IOException ex) { @@ -181,7 +180,7 @@ private OMResponse processRequest(OMRequest request) throws // To validate credentials we have already verified leader status. // This will skip of checking leader status again if request has S3Auth. if (!s3Auth) { - checkLeaderStatus(); + OzoneManagerRatisUtils.checkLeaderStatus(ozoneManager); } try { omClientRequest = @@ -267,7 +266,8 @@ private ServiceException createLeaderNotReadyException() { /** * Submits request directly to OM. */ - private OMResponse submitRequestDirectlyToOM(OMRequest request) { + private OMResponse submitRequestDirectlyToOM(OMRequest request) throws + ServiceException { OMClientResponse omClientResponse = null; long index = 0L; try { @@ -312,11 +312,6 @@ private OMResponse submitRequestDirectlyToOM(OMRequest request) { return omClientResponse.getOMResponse(); } - private void checkLeaderStatus() throws ServiceException { - OzoneManagerRatisUtils.checkLeaderStatus(omRatisServer.checkLeaderStatus(), - omRatisServer.getRaftPeerId()); - } - /** * Create OMResponse from the specified OMRequest and exception. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java index 10dda299793a..3e023d1f92ec 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java @@ -36,9 +36,12 @@ import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.S3SecretManager; import org.apache.hadoop.ozone.om.S3SecretManagerImpl; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.exceptions.OMLeaderNotReadyException; +import org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException; import org.apache.hadoop.ozone.security.OzoneSecretStore.OzoneManagerSecretState; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier.TokenInfo; import org.apache.hadoop.security.AccessControlException; @@ -71,6 +74,7 @@ public class OzoneDelegationTokenSecretManager private final long tokenRemoverScanInterval; private String omCertificateSerialId; private String omServiceId; + private final OzoneManager ozoneManager; /** * If the delegation token update thread holds this lock, it will not get @@ -94,6 +98,7 @@ public OzoneDelegationTokenSecretManager(Builder b) throws IOException { this.s3SecretManager = (S3SecretManagerImpl) b.s3SecretManager; this.store = new OzoneSecretStore(b.ozoneConf, this.s3SecretManager.getOmMetadataManager()); + this.ozoneManager = b.ozoneManager; isRatisEnabled = b.ozoneConf.getBoolean( OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, OMConfigKeys.OZONE_OM_RATIS_ENABLE_DEFAULT); @@ -113,6 +118,7 @@ public static class Builder { private S3SecretManager s3SecretManager; private CertificateClient certClient; private String omServiceId; + private OzoneManager ozoneManager; public OzoneDelegationTokenSecretManager build() throws IOException { return new OzoneDelegationTokenSecretManager(this); @@ -157,6 +163,11 @@ public Builder setOmServiceId(String serviceId) { this.omServiceId = serviceId; return this; } + + public Builder setOzoneManager(OzoneManager ozoneMgr) { + this.ozoneManager = ozoneMgr; + return this; + } } @Override @@ -404,6 +415,18 @@ public void removeToken(OzoneTokenIdentifier ozoneTokenIdentifier) { @Override public byte[] retrievePassword(OzoneTokenIdentifier identifier) throws InvalidToken { + // Tokens are a bit different in that a follower OM may be behind and + // thus not yet know of all tokens issued by the leader OM. the + // following check does not allow ANY token auth. In optimistic, it should + // allow known tokens in. + try { + ozoneManager.checkLeaderStatus(); + } catch (OMNotLeaderException | OMLeaderNotReadyException e) { + InvalidToken wrappedStandby = new InvalidToken("IOException"); + wrappedStandby.initCause(e); + throw wrappedStandby; + } + if (identifier.getTokenType().equals(S3AUTHINFO)) { return validateS3AuthInfo(identifier); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java index 377eec847a39..0c45c12f2507 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/S3SecurityUtil.java @@ -17,11 +17,14 @@ package org.apache.hadoop.ozone.security; +import com.google.protobuf.ServiceException; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.exceptions.OMLeaderNotReadyException; +import org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; import org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB; @@ -44,11 +47,12 @@ private S3SecurityUtil() { /** * Validate S3 Credentials which are part of {@link OMRequest}. * - * If validation is successful returns, else throw {@link OMException} - * @throws OMException + * If validation is successful returns, else throw exception. + * @throws OMException validation failure + * ServiceException Server is not leader or not ready */ public static void validateS3Credential(OMRequest omRequest, - OzoneManager ozoneManager) throws OMException { + OzoneManager ozoneManager) throws ServiceException, OMException { if (ozoneManager.isSecurityEnabled()) { OzoneTokenIdentifier s3Token = constructS3Token(omRequest); try { @@ -56,7 +60,13 @@ public static void validateS3Credential(OMRequest omRequest, // delegationTokenMgr validateToken via retrievePassword ozoneManager.getDelegationTokenMgr().retrievePassword(s3Token); } catch (SecretManager.InvalidToken e) { - // TODO: Just check are we okay to log enitre token in failure case. + if (e.getCause() != null && + (e.getCause().getClass() == OMNotLeaderException.class || + e.getCause().getClass() == OMLeaderNotReadyException.class)) { + throw new ServiceException(e.getCause()); + } + + // TODO: Just check are we okay to log entire token in failure case. OzoneManagerProtocolServerSideTranslatorPB.getLog().error( "signatures do NOT match for S3 identifier:{}", s3Token, e); throw new OMException("User " + s3Token.getAwsAccessId() diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java index e39fe39930c7..4814e8f783b3 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java @@ -37,8 +37,11 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.S3SecretManager; import org.apache.hadoop.ozone.om.S3SecretManagerImpl; +import org.apache.hadoop.ozone.om.exceptions.OMLeaderNotReadyException; +import org.apache.hadoop.ozone.om.exceptions.OMNotLeaderException; import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; @@ -55,12 +58,14 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; /** * Test class for {@link OzoneDelegationTokenSecretManager}. */ public class TestOzoneDelegationTokenSecretManager { + private OzoneManager om; private OzoneDelegationTokenSecretManager secretManager; private SecurityConfig securityConfig; private CertificateClient certificateClient; @@ -87,6 +92,7 @@ public void setUp() throws Exception { final Map s3Secrets = new HashMap<>(); s3Secrets.put("testuser1", s3Secret); s3Secrets.put("abc", "djakjahkd"); + om = Mockito.mock(OzoneManager.class); OMMetadataManager metadataManager = new OmMetadataManagerImpl(conf); s3SecretManager = new S3SecretManagerImpl(conf, metadataManager) { @Override @@ -161,6 +167,39 @@ public void tearDown() throws IOException { secretManager.stop(); } + @Test + public void testLeadershipCheckinRetrievePassword() throws Exception { + secretManager = createSecretManager(conf, tokenMaxLifetime, + expiryTime, tokenRemoverScanInterval); + Mockito.doThrow(new OMNotLeaderException("Not leader")) + .when(om).checkLeaderStatus(); + OzoneTokenIdentifier identifier = new OzoneTokenIdentifier(); + try { + secretManager.retrievePassword(identifier); + } catch (Exception e) { + Assert.assertEquals(SecretManager.InvalidToken.class, e.getClass()); + Assert.assertEquals(OMNotLeaderException.class, e.getCause().getClass()); + } + + Mockito.doThrow(new OMLeaderNotReadyException("Leader not ready")) + .when(om).checkLeaderStatus(); + try { + secretManager.retrievePassword(identifier); + } catch (Exception e) { + Assert.assertEquals(SecretManager.InvalidToken.class, e.getClass()); + Assert.assertEquals(OMLeaderNotReadyException.class, + e.getCause().getClass()); + } + + Mockito.doNothing().when(om).checkLeaderStatus(); + try { + secretManager.retrievePassword(identifier); + } catch (Exception e) { + Assert.assertEquals(SecretManager.InvalidToken.class, e.getClass()); + Assert.assertNull(e.getCause()); + } + } + @Test public void testCreateToken() throws Exception { secretManager = createSecretManager(conf, tokenMaxLifetime, @@ -413,6 +452,7 @@ private void validateHash(byte[] hash, byte[] identifier) throws Exception { .setTokenRenewInterval(expiry) .setTokenRemoverScanInterval(tokenRemoverScanTime) .setService(serviceRpcAdd) + .setOzoneManager(om) .setS3SecretManager(s3SecretManager) .setCertificateClient(certificateClient) .setOmServiceId(OzoneConsts.OM_SERVICE_ID_DEFAULT)