diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java index 64772553e028..cb258dfa74dc 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java @@ -193,6 +193,8 @@ public final class HddsConfigKeys { "hdds.x509.renew.grace.duration"; public static final String HDDS_X509_RENEW_GRACE_DURATION_DEFAULT = "P28D"; + public static final String HDDS_NEW_KEY_CERT_DIR_NAME_SUFFIX = "-next"; + public static final String HDDS_BACKUP_KEY_CERT_DIR_NAME_SUFFIX = "-previous"; public static final String HDDS_CONTAINER_REPLICATION_COMPRESSION = "hdds.container.replication.compression"; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java index 03e4c53da826..6e590df04898 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CertificateCodec.java @@ -79,6 +79,11 @@ public CertificateCodec(SecurityConfig config, String component) { this.location = securityConfig.getCertificateLocation(component); } + public CertificateCodec(SecurityConfig config, Path certPath) { + this.securityConfig = config; + this.location = certPath; + } + /** * Returns a X509 Certificate from the Certificate Holder. * diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 2d26d1e36525..62302d04ce9f 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -21,8 +21,6 @@ import java.io.File; import java.io.IOException; import java.net.InetAddress; -import java.security.KeyPair; -import java.security.cert.CertificateException; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -38,13 +36,10 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; -import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStore; import org.apache.hadoop.hdds.datanode.metadata.DatanodeCRLStoreImpl; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto; -import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient; @@ -72,14 +67,11 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import static org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec.getX509Certificate; -import static org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString; import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY; import static org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY; import static org.apache.hadoop.ozone.common.Storage.StorageState.INITIALIZED; import static org.apache.hadoop.util.ExitUtil.terminate; -import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; @@ -98,6 +90,7 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin { HddsDatanodeService.class); private OzoneConfiguration conf; + private SecurityConfig secConf; private DatanodeDetails datanodeDetails; private DatanodeStateMachine datanodeStateMachine; private List plugins; @@ -237,8 +230,10 @@ public void start() { if (OzoneSecurityUtil.isSecurityEnabled(conf)) { component = "dn-" + datanodeDetails.getUuidString(); - dnCertClient = new DNCertificateClient(new SecurityConfig(conf), - datanodeDetails.getCertSerialId()); + secConf = new SecurityConfig(conf); + dnCertClient = new DNCertificateClient(secConf, datanodeDetails, + datanodeDetails.getCertSerialId(), this::saveNewCertId, + this::terminateDatanode); if (SecurityUtil.getAuthenticationMethod(conf).equals( UserGroupInformation.AuthenticationMethod.KERBEROS)) { @@ -273,7 +268,7 @@ public void start() { dnCRLStore = new DatanodeCRLStoreImpl(conf); if (OzoneSecurityUtil.isSecurityEnabled(conf)) { - initializeCertificateClient(conf); + dnCertClient = initializeCertificateClient(dnCertClient); } datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf, dnCertClient, this::terminateDatanode, dnCRLStore); @@ -333,15 +328,16 @@ private void startRatisForTest() throws IOException { * Initializes secure Datanode. * */ @VisibleForTesting - public void initializeCertificateClient(OzoneConfiguration config) - throws IOException { + public CertificateClient initializeCertificateClient( + CertificateClient certClient) throws IOException { LOG.info("Initializing secure Datanode."); - CertificateClient.InitResponse response = dnCertClient.init(); + CertificateClient.InitResponse response = certClient.init(); if (response.equals(CertificateClient.InitResponse.REINIT)) { LOG.info("Re-initialize certificate client."); - dnCertClient = new DNCertificateClient(new SecurityConfig(conf)); - response = dnCertClient.init(); + certClient = new DNCertificateClient(secConf, datanodeDetails, null, + this::saveNewCertId, this::terminateDatanode); + response = certClient.init(); } LOG.info("Init response: {}", response); switch (response) { @@ -349,7 +345,14 @@ public void initializeCertificateClient(OzoneConfiguration config) LOG.info("Initialization successful, case:{}.", response); break; case GETCERT: - getSCMSignedCert(config); + CertificateSignRequest.Builder csrBuilder = certClient.getCSRBuilder(); + String dnCertSerialId = + certClient.signAndStoreCertificate(csrBuilder.build()); + // persist cert ID to VERSION file + datanodeDetails.setCertSerialId(dnCertSerialId); + persistDatanodeDetails(datanodeDetails); + // set new certificate ID + certClient.setCertificateId(dnCertSerialId); LOG.info("Successfully stored SCM signed certificate, case:{}.", response); break; @@ -365,51 +368,8 @@ public void initializeCertificateClient(OzoneConfiguration config) response); throw new RuntimeException("DN security initialization failed."); } - } - /** - * Get SCM signed certificate and store it using certificate client. - * @param config - * */ - private void getSCMSignedCert(OzoneConfiguration config) { - try { - PKCS10CertificationRequest csr = getCSR(config); - // TODO: For SCM CA we should fetch certificate from multiple SCMs. - SCMSecurityProtocolClientSideTranslatorPB secureScmClient = - HddsServerUtil.getScmSecurityClientWithMaxRetry(config); - SCMGetCertResponseProto response = secureScmClient. - getDataNodeCertificateChain( - datanodeDetails.getProtoBufMessage(), - getEncodedString(csr)); - // Persist certificates. - if (response.hasX509CACertificate()) { - String pemEncodedCert = response.getX509Certificate(); - dnCertClient.storeCertificate(pemEncodedCert, true); - dnCertClient.storeCertificate(response.getX509CACertificate(), true, - true); - - // Store Root CA certificate. - if (response.hasX509RootCACertificate()) { - dnCertClient.storeRootCACertificate( - response.getX509RootCACertificate(), true); - } - String dnCertSerialId = getX509Certificate(pemEncodedCert). - getSerialNumber().toString(); - datanodeDetails.setCertSerialId(dnCertSerialId); - persistDatanodeDetails(datanodeDetails); - // Rebuild dnCertClient with the new CSR result so that the default - // certSerialId and the x509Certificate can be updated. - dnCertClient = new DNCertificateClient( - new SecurityConfig(config), dnCertSerialId); - - } else { - throw new RuntimeException("Unable to retrieve datanode certificate " + - "chain"); - } - } catch (IOException | CertificateException e) { - LOG.error("Error while storing SCM signed certificate.", e); - throw new RuntimeException(e); - } + return certClient; } private void registerMXBean() { @@ -427,30 +387,6 @@ private void unregisterMXBean() { } } - /** - * Creates CSR for DN. - * @param config - * */ - @VisibleForTesting - public PKCS10CertificationRequest getCSR(ConfigurationSource config) - throws IOException { - CertificateSignRequest.Builder builder = dnCertClient.getCSRBuilder(); - KeyPair keyPair = new KeyPair(dnCertClient.getPublicKey(), - dnCertClient.getPrivateKey()); - - String hostname = InetAddress.getLocalHost().getCanonicalHostName(); - String subject = UserGroupInformation.getCurrentUser() - .getShortUserName() + "@" + hostname; - - builder.setCA(false) - .setKey(keyPair) - .setConfiguration(config) - .setSubject(subject); - - LOG.info("Creating csr for DN-> subject:{}", subject); - return builder.build(); - } - /** * Returns DatanodeDetails or null in case of Error. * @@ -584,7 +520,9 @@ public void stop() { unregisterMXBean(); // stop dn crl store try { - dnCRLStore.stop(); + if (dnCRLStore != null) { + dnCRLStore.stop(); + } } catch (Exception ex) { LOG.error("Datanode CRL store stop failed", ex); } @@ -631,4 +569,18 @@ public void setCertificateClient(CertificateClient client) { public void printError(Throwable error) { LOG.error("Exception in HddsDatanodeService.", error); } + + public void saveNewCertId(String newCertId) { + // save new certificate Id to VERSION file + datanodeDetails.setCertSerialId(newCertId); + try { + persistDatanodeDetails(datanodeDetails); + } catch (IOException ex) { + // New cert ID cannot be persisted into VERSION file. + String msg = "Failed to persist new cert ID " + newCertId + + "to VERSION file. Terminating datanode..."; + LOG.error(msg, ex); + terminateDatanode(); + } + } } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java index 39e36baae02b..00f9cb0dd92f 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java @@ -21,17 +21,23 @@ import java.security.KeyPair; import java.security.PrivateKey; import java.security.PublicKey; -import java.security.cert.X509Certificate; +import java.security.cert.CertificateExpiredException; +import java.time.Duration; +import java.time.LocalDateTime; import java.util.concurrent.Callable; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.DFSConfigKeysLegacy; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos; +import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.security.x509.SecurityConfig; -import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; +import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate; import org.apache.hadoop.hdds.security.x509.keys.KeyCodec; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.ozone.test.GenericTestUtils; @@ -39,10 +45,18 @@ import org.apache.hadoop.util.ServicePlugin; import org.apache.commons.io.FileUtils; + +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION; import static org.apache.hadoop.ozone.HddsDatanodeService.getLogger; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; +import static org.mockito.ArgumentMatchers.anyObject; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import org.bouncycastle.cert.X509CertificateHolder; import org.bouncycastle.pkcs.PKCS10CertificationRequest; +import org.junit.Assert; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -53,7 +67,6 @@ * Test class for {@link HddsDatanodeService}. */ public class TestHddsSecureDatanodeInit { - private static File testDir; private static OzoneConfiguration conf; private static HddsDatanodeService service; @@ -66,8 +79,10 @@ public class TestHddsSecureDatanodeInit { private static CertificateCodec certCodec; private static X509CertificateHolder certHolder; private static final String DN_COMPONENT = DNCertificateClient.COMPONENT_NAME; + private static final int CERT_LIFETIME = 15; // seconds - private CertificateClient client; + private DNCertificateClient client; + private static DatanodeDetails datanodeDetails; @BeforeAll public static void setUp() throws Exception { @@ -82,6 +97,7 @@ public static void setUp() throws Exception { conf.setClass(OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY, TestHddsDatanodeService.MockService.class, ServicePlugin.class); + conf.set(HDDS_X509_RENEW_GRACE_DURATION, "PT5S"); // 5s securityConfig = new SecurityConfig(conf); service = HddsDatanodeService.createHddsDatanodeService(args); @@ -91,7 +107,7 @@ public static void setUp() throws Exception { return null; }); callQuietly(() -> { - service.initializeCertificateClient(conf); + service.initializeCertificateClient(service.getCertificateClient()); return null; }); certCodec = new CertificateCodec(securityConfig, DN_COMPONENT); @@ -99,13 +115,10 @@ public static void setUp() throws Exception { dnLogs.clearOutput(); privateKey = service.getCertificateClient().getPrivateKey(); publicKey = service.getCertificateClient().getPublicKey(); - X509Certificate x509Certificate = null; - - x509Certificate = KeyStoreTestUtil.generateCertificate( - "CN=Test", new KeyPair(publicKey, privateKey), 365, - securityConfig.getSignatureAlgo()); - certHolder = new X509CertificateHolder(x509Certificate.getEncoded()); + certHolder = generateX509CertHolder(new KeyPair(publicKey, privateKey), + null, Duration.ofSeconds(CERT_LIFETIME)); + datanodeDetails = MockDatanodeDetails.randomDatanodeDetails(); } @AfterAll @@ -126,8 +139,8 @@ public void setUpDNCertClient() { .getCertificateLocation(DN_COMPONENT).toString(), securityConfig.getCertificateFileName()).toFile()); dnLogs.clearOutput(); - client = new DNCertificateClient(securityConfig, - certHolder.getSerialNumber().toString()); + client = new DNCertificateClient(securityConfig, datanodeDetails, + certHolder.getSerialNumber().toString(), null, null); service.setCertificateClient(client); } @@ -137,7 +150,7 @@ public void testSecureDnStartupCase0() throws Exception { // Case 0: When keypair as well as certificate is missing. Initial keypair // boot-up. Get certificate will fail as no SCM is not running. LambdaTestUtils.intercept(Exception.class, "", - () -> service.initializeCertificateClient(conf)); + () -> service.initializeCertificateClient(client)); Assertions.assertNotNull(client.getPrivateKey()); Assertions.assertNotNull(client.getPublicKey()); @@ -153,7 +166,7 @@ public void testSecureDnStartupCase1() throws Exception { certCodec.writeCertificate(certHolder); LambdaTestUtils.intercept(RuntimeException.class, "DN security" + " initialization failed", - () -> service.initializeCertificateClient(conf)); + () -> service.initializeCertificateClient(client)); Assertions.assertNull(client.getPrivateKey()); Assertions.assertNull(client.getPublicKey()); Assertions.assertNotNull(client.getCertificate()); @@ -167,7 +180,7 @@ public void testSecureDnStartupCase2() throws Exception { keyCodec.writePublicKey(publicKey); LambdaTestUtils.intercept(RuntimeException.class, "DN security" + " initialization failed", - () -> service.initializeCertificateClient(conf)); + () -> service.initializeCertificateClient(client)); Assertions.assertNull(client.getPrivateKey()); Assertions.assertNotNull(client.getPublicKey()); Assertions.assertNull(client.getCertificate()); @@ -182,7 +195,7 @@ public void testSecureDnStartupCase3() throws Exception { certCodec.writeCertificate(certHolder); LambdaTestUtils.intercept(RuntimeException.class, "DN security" + " initialization failed", - () -> service.initializeCertificateClient(conf)); + () -> service.initializeCertificateClient(client)); Assertions.assertNull(client.getPrivateKey()); Assertions.assertNotNull(client.getPublicKey()); Assertions.assertNotNull(client.getCertificate()); @@ -196,7 +209,7 @@ public void testSecureDnStartupCase4() throws Exception { keyCodec.writePrivateKey(privateKey); LambdaTestUtils.intercept(RuntimeException.class, " DN security" + " initialization failed", - () -> service.initializeCertificateClient(conf)); + () -> service.initializeCertificateClient(client)); Assertions.assertNotNull(client.getPrivateKey()); Assertions.assertNull(client.getPublicKey()); Assertions.assertNull(client.getCertificate()); @@ -210,7 +223,7 @@ public void testSecureDnStartupCase5() throws Exception { // Case 5: If private key and certificate is present. certCodec.writeCertificate(certHolder); keyCodec.writePrivateKey(privateKey); - service.initializeCertificateClient(conf); + service.initializeCertificateClient(client); Assertions.assertNotNull(client.getPrivateKey()); Assertions.assertNotNull(client.getPublicKey()); Assertions.assertNotNull(client.getCertificate()); @@ -224,7 +237,7 @@ public void testSecureDnStartupCase6() throws Exception { keyCodec.writePublicKey(publicKey); keyCodec.writePrivateKey(privateKey); LambdaTestUtils.intercept(Exception.class, "", - () -> service.initializeCertificateClient(conf)); + () -> service.initializeCertificateClient(client)); Assertions.assertNotNull(client.getPrivateKey()); Assertions.assertNotNull(client.getPublicKey()); Assertions.assertNull(client.getCertificate()); @@ -239,7 +252,7 @@ public void testSecureDnStartupCase7() throws Exception { keyCodec.writePrivateKey(privateKey); certCodec.writeCertificate(certHolder); - service.initializeCertificateClient(conf); + service.initializeCertificateClient(client); Assertions.assertNotNull(client.getPrivateKey()); Assertions.assertNotNull(client.getPublicKey()); Assertions.assertNotNull(client.getCertificate()); @@ -266,17 +279,175 @@ public void testGetCSR() throws Exception { keyCodec.writePrivateKey(privateKey); service.setCertificateClient(client); PKCS10CertificationRequest csr = - service.getCSR(conf); + client.getCSRBuilder().build(); Assertions.assertNotNull(csr); - csr = service.getCSR(conf); + csr = client.getCSRBuilder().build(); Assertions.assertNotNull(csr); - csr = service.getCSR(conf); + csr = client.getCSRBuilder().build(); Assertions.assertNotNull(csr); - csr = service.getCSR(conf); + csr = client.getCSRBuilder().build(); Assertions.assertNotNull(csr); } + @Test + public void testCertificateRotation() throws Exception { + // save the certificate on dn + certCodec.writeCertificate(certHolder); + + // prepare a mocked scmClient to certificate signing + SCMSecurityProtocolClientSideTranslatorPB scmClient = + mock(SCMSecurityProtocolClientSideTranslatorPB.class); + client.setSecureScmClient(scmClient); + + Duration gracePeriod = securityConfig.getRenewalGracePeriod(); + X509CertificateHolder newCertHolder = generateX509CertHolder(null, + LocalDateTime.now().plus(gracePeriod), + Duration.ofSeconds(CERT_LIFETIME)); + String pemCert = CertificateCodec.getPEMEncodedString(newCertHolder); + SCMSecurityProtocolProtos.SCMGetCertResponseProto responseProto = + SCMSecurityProtocolProtos.SCMGetCertResponseProto + .newBuilder().setResponseCode(SCMSecurityProtocolProtos + .SCMGetCertResponseProto.ResponseCode.success) + .setX509Certificate(pemCert) + .setX509CACertificate(pemCert) + .setX509RootCACertificate(pemCert) + .build(); + when(scmClient.getDataNodeCertificateChain(anyObject(), anyString())) + .thenReturn(responseProto); + + // check that new cert ID should not equal to current cert ID + String certId = newCertHolder.getSerialNumber().toString(); + Assert.assertFalse(certId.equals( + client.getCertificate().getSerialNumber().toString())); + + // start monitor task to renew key and cert + client.startCertificateMonitor(); + + // check after renew, client will have the new cert ID + GenericTestUtils.waitFor(() -> { + String newCertId = client.getCertificate().getSerialNumber().toString(); + return newCertId.equals(certId); + }, 1000, CERT_LIFETIME * 1000); + PrivateKey privateKey1 = client.getPrivateKey(); + PublicKey publicKey1 = client.getPublicKey(); + String caCertId1 = client.getCACertificate().getSerialNumber().toString(); + String rootCaCertId1 = + client.getRootCACertificate().getSerialNumber().toString(); + + // test the second time certificate rotation, generate a new cert + newCertHolder = generateX509CertHolder(null, null, + Duration.ofSeconds(CERT_LIFETIME)); + pemCert = CertificateCodec.getPEMEncodedString(newCertHolder); + responseProto = SCMSecurityProtocolProtos.SCMGetCertResponseProto + .newBuilder().setResponseCode(SCMSecurityProtocolProtos + .SCMGetCertResponseProto.ResponseCode.success) + .setX509Certificate(pemCert) + .setX509CACertificate(pemCert) + .setX509RootCACertificate(pemCert) + .build(); + when(scmClient.getDataNodeCertificateChain(anyObject(), anyString())) + .thenReturn(responseProto); + String certId2 = newCertHolder.getSerialNumber().toString(); + + // check after renew, client will have the new cert ID + GenericTestUtils.waitFor(() -> { + String newCertId = client.getCertificate().getSerialNumber().toString(); + return newCertId.equals(certId2); + }, 1000, CERT_LIFETIME * 1000); + Assert.assertFalse(client.getPrivateKey().equals(privateKey1)); + Assert.assertFalse(client.getPublicKey().equals(publicKey1)); + Assert.assertFalse(client.getCACertificate().getSerialNumber() + .toString().equals(caCertId1)); + Assert.assertFalse(client.getRootCACertificate().getSerialNumber() + .toString().equals(rootCaCertId1)); + } + + /** + * Test unexpected SCMGetCertResponseProto returned from SCM. + */ + @Test + public void testCertificateRotationRecoverableFailure() throws Exception { + // save the certificate on dn + certCodec.writeCertificate(certHolder); + + // prepare a mocked scmClient to certificate signing + SCMSecurityProtocolClientSideTranslatorPB scmClient = + mock(SCMSecurityProtocolClientSideTranslatorPB.class); + client.setSecureScmClient(scmClient); + + Duration gracePeriod = securityConfig.getRenewalGracePeriod(); + X509CertificateHolder newCertHolder = generateX509CertHolder(null, + LocalDateTime.now().plus(gracePeriod), + Duration.ofSeconds(CERT_LIFETIME)); + String pemCert = CertificateCodec.getPEMEncodedString(newCertHolder); + // provide an invalid SCMGetCertResponseProto. Without + // setX509CACertificate(pemCert), signAndStoreCert will throw exception. + SCMSecurityProtocolProtos.SCMGetCertResponseProto responseProto = + SCMSecurityProtocolProtos.SCMGetCertResponseProto + .newBuilder().setResponseCode(SCMSecurityProtocolProtos + .SCMGetCertResponseProto.ResponseCode.success) + .setX509Certificate(pemCert) + .build(); + when(scmClient.getDataNodeCertificateChain(anyObject(), anyString())) + .thenReturn(responseProto); + + // check that new cert ID should not equal to current cert ID + String certId = newCertHolder.getSerialNumber().toString(); + Assert.assertFalse(certId.equals( + client.getCertificate().getSerialNumber().toString())); + + // start monitor task to renew key and cert + client.startCertificateMonitor(); + + // certificate failed to renew, client still hold the old expired cert. + Thread.sleep(CERT_LIFETIME * 1000); + Assert.assertFalse(certId.equals( + client.getCertificate().getSerialNumber().toString())); + try { + client.getCertificate().checkValidity(); + } catch (Exception e) { + Assert.assertTrue(e instanceof CertificateExpiredException); + } + + // provide a new valid SCMGetCertResponseProto + newCertHolder = generateX509CertHolder(null, null, + Duration.ofSeconds(CERT_LIFETIME)); + pemCert = CertificateCodec.getPEMEncodedString(newCertHolder); + responseProto = SCMSecurityProtocolProtos.SCMGetCertResponseProto + .newBuilder().setResponseCode(SCMSecurityProtocolProtos + .SCMGetCertResponseProto.ResponseCode.success) + .setX509Certificate(pemCert) + .setX509CACertificate(pemCert) + .build(); + when(scmClient.getDataNodeCertificateChain(anyObject(), anyString())) + .thenReturn(responseProto); + String certId2 = newCertHolder.getSerialNumber().toString(); + + // check after renew, client will have the new cert ID + GenericTestUtils.waitFor(() -> { + String newCertId = client.getCertificate().getSerialNumber().toString(); + return newCertId.equals(certId2); + }, 1000, CERT_LIFETIME * 1000); + } + + private static X509CertificateHolder generateX509CertHolder(KeyPair keyPair, + LocalDateTime startDate, Duration certLifetime) throws Exception { + if (keyPair == null) { + keyPair = KeyStoreTestUtil.generateKeyPair("RSA"); + } + LocalDateTime start = startDate == null ? LocalDateTime.now() : startDate; + LocalDateTime end = start.plus(certLifetime); + return SelfSignedCertificate.newBuilder() + .setBeginDate(start) + .setEndDate(end) + .setClusterID("cluster") + .setKey(keyPair) + .setSubject("localhost") + .setConfiguration(conf) + .setScmID("test") + .build(); + } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/MonitoringTimerTask.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/MonitoringTimerTask.java index d6dd1ff7f7c8..392b74195803 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/MonitoringTimerTask.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/MonitoringTimerTask.java @@ -64,15 +64,13 @@ public MonitoringTimerTask(CertificateClient caClient, @Override public void run() { - if (caClient.isCertificateRenewed()) { - try { - onReload.accept(caClient); - } catch (Throwable t) { - if (onReloadFailure != null) { - onReloadFailure.accept(t); - } else { - LOG.error(PROCESS_ERROR_MESSAGE, t); - } + try { + onReload.accept(caClient); + } catch (Throwable t) { + if (onReloadFailure != null) { + onReloadFailure.accept(t); + } else { + LOG.error(PROCESS_ERROR_MESSAGE, t); } } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java index ce7cb0485552..641b26edb1a2 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java @@ -24,10 +24,13 @@ import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.crl.CRLInfo; import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Path; +import java.security.KeyPair; import java.security.PrivateKey; import java.security.PublicKey; import java.security.cert.CertStore; @@ -77,15 +80,6 @@ X509Certificate getCertificate(String certSerialId) */ X509Certificate getCertificate(); - /** - * Returns whether certificate of the specified component is renewed. - * - * @return true if it's renewed recently. - */ - default boolean isCertificateRenewed() { - return false; - } - /** * Return the latest CA certificate known to the client. * @return latest ca certificate known to the client. @@ -99,6 +93,12 @@ default boolean isCertificateRenewed() { */ boolean verifyCertificate(X509Certificate certificate); + /** + * Set the serial ID of default certificate for the specified component. + * @param certSerialId - certificate ID. + * */ + void setCertificateId(String certSerialId); + /** * Creates digital signature over the data stream using the components private * key. @@ -141,7 +141,35 @@ boolean verifySignature(byte[] data, byte[] signature, * * @return CertificateSignRequest.Builder */ - CertificateSignRequest.Builder getCSRBuilder() throws CertificateException; + CertificateSignRequest.Builder getCSRBuilder(KeyPair keyPair) + throws IOException; + + /** + * Returns a CSR builder that can be used to create a Certificate sigining + * request. + * + * @return CertificateSignRequest.Builder + */ + CertificateSignRequest.Builder getCSRBuilder() + throws CertificateException; + + /** + * Send request to SCM to sign the certificate and save certificates returned + * by SCM to PEM files on disk. + * + * @return the serial ID of the new certificate + */ + String signAndStoreCertificate(PKCS10CertificationRequest request, + Path certPath) throws CertificateException; + + /** + * Send request to SCM to sign the certificate and save certificates returned + * by SCM to PEM files on disk. + * + * @return the serial ID of the new certificate + */ + String signAndStoreCertificate(PKCS10CertificationRequest request) + throws CertificateException; /** * Get the certificate of well-known entity from SCM. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CommonCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CommonCertificateClient.java index bb122955a526..32b2cbb32c01 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CommonCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CommonCertificateClient.java @@ -22,6 +22,8 @@ import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; import org.slf4j.Logger; +import java.util.function.Consumer; + import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.FAILURE; import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.GETCERT; import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.RECOVER; @@ -31,13 +33,15 @@ /** * Common Certificate client. */ -public class CommonCertificateClient extends DefaultCertificateClient { +public abstract class CommonCertificateClient extends DefaultCertificateClient { private final Logger log; public CommonCertificateClient(SecurityConfig securityConfig, Logger log, - String certSerialId, String component) { - super(securityConfig, log, certSerialId, component); + String certSerialId, String component, + Consumer saveCertIdCallback, Runnable shutdownCallback) { + super(securityConfig, log, certSerialId, component, saveCertIdCallback, + shutdownCallback); this.log = log; } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DNCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DNCertificateClient.java index 40c5b0a7317a..613cca435546 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DNCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DNCertificateClient.java @@ -19,12 +19,26 @@ package org.apache.hadoop.hdds.security.x509.certificate.client; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos; +import org.apache.hadoop.hdds.security.x509.SecurityConfig; +import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; +import org.apache.hadoop.security.UserGroupInformation; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hdds.security.x509.SecurityConfig; +import java.io.IOException; +import java.net.InetAddress; +import java.nio.file.Path; +import java.security.KeyPair; +import java.util.function.Consumer; + +import static org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec.getX509Certificate; +import static org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString; +import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateException.ErrorCode.CSR_ERROR; /** * Certificate client for DataNodes. @@ -35,14 +49,27 @@ public class DNCertificateClient extends DefaultCertificateClient { LoggerFactory.getLogger(DNCertificateClient.class); public static final String COMPONENT_NAME = "dn"; + private final DatanodeDetails dn; public DNCertificateClient(SecurityConfig securityConfig, - String certSerialId) { - super(securityConfig, LOG, certSerialId, COMPONENT_NAME); + DatanodeDetails datanodeDetails, String certSerialId, + Consumer saveCertId, Runnable shutdown) { + super(securityConfig, LOG, certSerialId, COMPONENT_NAME, + saveCertId, shutdown); + this.dn = datanodeDetails; } - public DNCertificateClient(SecurityConfig securityConfig) { - super(securityConfig, LOG, null, COMPONENT_NAME); + /** + * Returns a CSR builder that can be used to creates a Certificate signing + * request. + * The default flag is added to allow basic SSL handshake. + * + * @return CertificateSignRequest.Builder + */ + @Override + public CertificateSignRequest.Builder getCSRBuilder() + throws CertificateException { + return getCSRBuilder(new KeyPair(getPublicKey(), getPrivateKey())); } /** @@ -53,11 +80,67 @@ public DNCertificateClient(SecurityConfig securityConfig) { * @return CertificateSignRequest.Builder */ @Override - public CertificateSignRequest.Builder getCSRBuilder() + public CertificateSignRequest.Builder getCSRBuilder(KeyPair keyPair) throws CertificateException { - return super.getCSRBuilder() + CertificateSignRequest.Builder builder = super.getCSRBuilder() .setDigitalEncryption(true) .setDigitalSignature(true); + + try { + String hostname = InetAddress.getLocalHost().getCanonicalHostName(); + String subject = UserGroupInformation.getCurrentUser() + .getShortUserName() + "@" + hostname; + builder.setCA(false) + .setKey(keyPair) + .setConfiguration(getConfig()) + .setSubject(subject); + + LOG.info("Created csr for DN-> subject:{}", subject); + return builder; + } catch (Exception e) { + LOG.error("Failed to get hostname or current user", e); + throw new CertificateException("Failed to get hostname or current user", + e, CSR_ERROR); + } + } + + @Override + public String signAndStoreCertificate(PKCS10CertificationRequest csr, + Path certPath) throws CertificateException { + try { + // TODO: For SCM CA we should fetch certificate from multiple SCMs. + SCMSecurityProtocolProtos.SCMGetCertResponseProto response = + getScmSecureClient().getDataNodeCertificateChain( + dn.getProtoBufMessage(), getEncodedString(csr)); + + // Persist certificates. + if (response.hasX509CACertificate()) { + String pemEncodedCert = response.getX509Certificate(); + CertificateCodec certCodec = new CertificateCodec( + getSecurityConfig(), certPath); + // Certs will be added to cert map after reloadAllCertificate called + storeCertificate(pemEncodedCert, true, false, false, certCodec, false); + storeCertificate(response.getX509CACertificate(), true, true, + false, certCodec, false); + + // Store Root CA certificate. + if (response.hasX509RootCACertificate()) { + storeCertificate(response.getX509RootCACertificate(), true, false, + true, certCodec, false); + } + // Return the default certificate ID + String dnCertSerialId = getX509Certificate(pemEncodedCert). + getSerialNumber().toString(); + return dnCertSerialId; + } else { + throw new CertificateException("Unable to retrieve datanode " + + "certificate chain."); + } + } catch (IOException | java.security.cert.CertificateException e) { + LOG.error("Error while signing and storing SCM signed certificate.", e); + throw new CertificateException( + "Error while signing and storing SCM signed certificate.", e); + } } @Override diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index 752113f7b347..8647c324f54d 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@ -26,6 +26,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.security.InvalidKeyException; import java.security.KeyPair; import java.security.NoSuchAlgorithmException; @@ -38,19 +40,29 @@ import java.security.cert.CertStore; import java.security.cert.X509Certificate; import java.security.spec.InvalidKeySpecException; +import java.time.Duration; import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.util.ArrayList; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Random; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; +import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory; import org.apache.hadoop.hdds.security.x509.crl.CRLInfo; import org.apache.hadoop.hdds.security.x509.SecurityConfig; @@ -68,6 +80,8 @@ import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.validator.routines.DomainValidator; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BACKUP_KEY_CERT_DIR_NAME_SUFFIX; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_NEW_KEY_CERT_DIR_NAME_SUFFIX; import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.FAILURE; import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.GETCERT; import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.REINIT; @@ -77,10 +91,12 @@ import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateException.ErrorCode.CRYPTO_SIGNATURE_VERIFICATION_ERROR; import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateException.ErrorCode.CRYPTO_SIGN_ERROR; import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateException.ErrorCode.CSR_ERROR; -import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClient; +import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateException.ErrorCode.RENEW_ERROR; +import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateException.ErrorCode.ROLLBACK_ERROR; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClientWithMaxRetry; import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; /** @@ -110,12 +126,26 @@ public abstract class DefaultCertificateClient implements CertificateClient { private long localCrlId; private String component; private List pemEncodedCACerts = null; - private final Lock lock; private KeyStoresFactory serverKeyStoresFactory; private KeyStoresFactory clientKeyStoresFactory; + // Lock to protect the certificate renew process, to make sure there is only + // one renew process is ongoing at one time. + // Certificate renew steps: + // 1. generate new keys and sign new certificate, persist all data to disk + // 2. switch on disk new keys and certificate with current ones + // 3. save new certificate ID into service VERSION file + // 4. refresh in memory certificate ID and reload all new certificates + private Lock renewLock = new ReentrantLock(); + + private ScheduledExecutorService executorService; + private Consumer certIdSaveCallback; + private Runnable shutdownCallback; + private SCMSecurityProtocolClientSideTranslatorPB scmSecurityProtocolClient; + DefaultCertificateClient(SecurityConfig securityConfig, Logger log, - String certSerialId, String component) { + String certSerialId, String component, + Consumer saveCertId, Runnable shutdown) { Objects.requireNonNull(securityConfig); this.securityConfig = securityConfig; keyCodec = new KeyCodec(securityConfig, component); @@ -123,15 +153,24 @@ public abstract class DefaultCertificateClient implements CertificateClient { this.certificateMap = new ConcurrentHashMap<>(); this.certSerialId = certSerialId; this.component = component; - lock = new ReentrantLock(); + this.certIdSaveCallback = saveCertId; + this.shutdownCallback = shutdown; + + loadAllCertificates(); + } + public synchronized void setCertificateId(String certId) { + Preconditions.checkArgument(certSerialId == null, + "certSerialId should only be set once if not renew"); + this.certSerialId = certId; + // reload all new certs loadAllCertificates(); } /** * Load all certificates from configured location. * */ - private void loadAllCertificates() { + private synchronized void loadAllCertificates() { // See if certs directory exists in file system. Path certPath = securityConfig.getCertificateLocation(component); if (Files.exists(certPath) && Files.isDirectory(certPath)) { @@ -176,7 +215,7 @@ private void loadAllCertificates() { latestRootCaCertSerialId = tmpRootCaCertSerailId; } } - getLogger().info("Added certificate from file:{}.", + getLogger().info("Added certificate {} from file:{}.", cert, file.getAbsolutePath()); } else { getLogger().error("Error reading certificate from file:{}", @@ -194,6 +233,15 @@ private void loadAllCertificates() { if (latestRootCaCertSerialId != -1) { rootCaCertId = Long.toString(latestRootCaCertSerialId); } + + if (x509Certificate != null) { + if (executorService == null) { + startCertificateMonitor(); + } + } else { + getLogger().warn("CertificateLifetimeMonitor is not started this " + + "time because certificate is empty."); + } } } } @@ -205,7 +253,7 @@ private void loadAllCertificates() { * @return private key or Null if there is no data. */ @Override - public PrivateKey getPrivateKey() { + public synchronized PrivateKey getPrivateKey() { if (privateKey != null) { return privateKey; } @@ -253,7 +301,7 @@ public PublicKey getPublicKey() { * @return certificate or Null if there is no data. */ @Override - public X509Certificate getCertificate() { + public synchronized X509Certificate getCertificate() { if (x509Certificate != null) { return x509Certificate; } @@ -276,7 +324,7 @@ public X509Certificate getCertificate() { * @return latest ca certificate known to the client. */ @Override - public X509Certificate getCACertificate() { + public synchronized X509Certificate getCACertificate() { if (caCertId != null) { return certificateMap.get(caCertId); } @@ -291,7 +339,7 @@ public X509Certificate getCACertificate() { * @return certificate or Null if there is no data. */ @Override - public X509Certificate getCertificate(String certId) + public synchronized X509Certificate getCertificate(String certId) throws CertificateException { // Check if it is in cache. if (certificateMap.containsKey(certId)) { @@ -304,9 +352,7 @@ public X509Certificate getCertificate(String certId) @Override public List getCrls(List crlIds) throws IOException { try { - SCMSecurityProtocol scmSecurityProtocolClient = getScmSecurityClient( - securityConfig.getConfiguration()); - return scmSecurityProtocolClient.getCrls(crlIds); + return getScmSecureClient().getCrls(crlIds); } catch (Exception e) { getLogger().error("Error while getting CRL with " + "CRL ids:{} from scm.", crlIds, e); @@ -318,9 +364,7 @@ public List getCrls(List crlIds) throws IOException { @Override public long getLatestCrlId() throws IOException { try { - SCMSecurityProtocol scmSecurityProtocolClient = getScmSecurityClient( - securityConfig.getConfiguration()); - return scmSecurityProtocolClient.getLatestCrlId(); + return getScmSecureClient().getLatestCrlId(); } catch (Exception e) { getLogger().error("Error while getting latest CRL id from scm.", e); throw new CertificateException("Error while getting latest CRL id from" + @@ -339,11 +383,7 @@ private X509Certificate getCertificateFromScm(String certId) getLogger().info("Getting certificate with certSerialId:{}.", certId); try { - SCMSecurityProtocol scmSecurityProtocolClient = - getScmSecurityClientWithMaxRetry( - (OzoneConfiguration) securityConfig.getConfiguration()); - String pemEncodedCert = - scmSecurityProtocolClient.getCertificate(certId); + String pemEncodedCert = getScmSecureClient().getCertificate(certId); this.storeCertificate(pemEncodedCert, true); return CertificateCodec.getX509Certificate(pemEncodedCert); } catch (Exception e) { @@ -583,22 +623,33 @@ public void storeCertificate(String pemEncodedCert, boolean force, boolean caCert) throws CertificateException { CertificateCodec certificateCodec = new CertificateCodec(securityConfig, component); - try { - Path basePath = securityConfig.getCertificateLocation(component); + storeCertificate(pemEncodedCert, force, caCert, false, + certificateCodec, true); + } + public synchronized void storeCertificate(String pemEncodedCert, + boolean force, boolean isCaCert, boolean isRootCaCert, + CertificateCodec codec, boolean addToCertMap) + throws CertificateException { + try { X509Certificate cert = CertificateCodec.getX509Certificate(pemEncodedCert); String certName = String.format(CERT_FILE_NAME_FORMAT, cert.getSerialNumber().toString()); - if (caCert) { + if (isCaCert) { certName = CA_CERT_PREFIX + certName; caCertId = cert.getSerialNumber().toString(); + } else if (isRootCaCert) { + certName = ROOT_CA_CERT_PREFIX + certName; + rootCaCertId = cert.getSerialNumber().toString(); } - certificateCodec.writeCertificate(basePath, certName, + codec.writeCertificate(codec.getLocation(), certName, pemEncodedCert, force); - certificateMap.putIfAbsent(cert.getSerialNumber().toString(), cert); + if (addToCertMap) { + certificateMap.putIfAbsent(cert.getSerialNumber().toString(), cert); + } } catch (IOException | java.security.cert.CertificateException e) { throw new CertificateException("Error while storing certificate.", e, CERTIFICATE_ERROR); @@ -612,7 +663,7 @@ public void storeCertificate(String pemEncodedCert, boolean force, * @throws CertificateException - on Error. */ @Override - public synchronized void storeTrustChain(CertStore ks) + public void storeTrustChain(CertStore ks) throws CertificateException { throw new UnsupportedOperationException("Operation not supported."); } @@ -625,7 +676,7 @@ public synchronized void storeTrustChain(CertStore ks) * @throws CertificateException - on Error. */ @Override - public synchronized void storeTrustChain(List certificates) + public void storeTrustChain(List certificates) throws CertificateException { throw new UnsupportedOperationException("Operation not supported."); } @@ -899,18 +950,18 @@ protected void bootstrapClientKeys() throws CertificateException { "for certificate storage.", BOOTSTRAP_ERROR); } } - KeyPair keyPair = createKeyPair(); + KeyPair keyPair = createKeyPair(keyCodec); privateKey = keyPair.getPrivate(); publicKey = keyPair.getPublic(); } - protected KeyPair createKeyPair() throws CertificateException { + protected KeyPair createKeyPair(KeyCodec codec) throws CertificateException { HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(securityConfig); KeyPair keyPair = null; try { keyPair = keyGenerator.generateKey(); - keyCodec.writePublicKey(keyPair.getPublic()); - keyCodec.writePrivateKey(keyPair.getPrivate()); + codec.writePublicKey(keyPair.getPublic()); + codec.writePrivateKey(keyPair.getPrivate()); } catch (NoSuchProviderException | NoSuchAlgorithmException | IOException e) { getLogger().error("Error while bootstrapping certificate client.", e); @@ -929,7 +980,7 @@ public String getComponentName() { } @Override - public X509Certificate getRootCACertificate() { + public synchronized X509Certificate getRootCACertificate() { if (rootCaCertId != null) { return certificateMap.get(rootCaCertId); } @@ -941,65 +992,32 @@ public void storeRootCACertificate(String pemEncodedCert, boolean force) throws CertificateException { CertificateCodec certificateCodec = new CertificateCodec(securityConfig, component); - try { - Path basePath = securityConfig.getCertificateLocation(component); - - X509Certificate cert = - CertificateCodec.getX509Certificate(pemEncodedCert); - String certName = String.format(CERT_FILE_NAME_FORMAT, - cert.getSerialNumber().toString()); - - certName = ROOT_CA_CERT_PREFIX + certName; - rootCaCertId = cert.getSerialNumber().toString(); - - certificateCodec.writeCertificate(basePath, certName, - pemEncodedCert, force); - certificateMap.putIfAbsent(cert.getSerialNumber().toString(), cert); - } catch (IOException | java.security.cert.CertificateException e) { - throw new CertificateException("Error while storing Root CA " + - "certificate.", e, CERTIFICATE_ERROR); - } + storeCertificate(pemEncodedCert, force, false, true, + certificateCodec, true); } @Override - public List getCAList() { - lock.lock(); - try { - return pemEncodedCACerts; - } finally { - lock.unlock(); - } + public synchronized List getCAList() { + return pemEncodedCACerts; } @Override - public List listCA() throws IOException { - lock.lock(); - try { - if (pemEncodedCACerts == null) { - updateCAList(); - } - return pemEncodedCACerts; - } finally { - lock.unlock(); + public synchronized List listCA() throws IOException { + if (pemEncodedCACerts == null) { + updateCAList(); } + return pemEncodedCACerts; } @Override - public List updateCAList() throws IOException { - lock.lock(); + public synchronized List updateCAList() throws IOException { try { - SCMSecurityProtocol scmSecurityProtocolClient = - getScmSecurityClientWithMaxRetry( - (OzoneConfiguration) securityConfig.getConfiguration()); - pemEncodedCACerts = - scmSecurityProtocolClient.listCACertificate(); + pemEncodedCACerts = getScmSecureClient().listCACertificate(); return pemEncodedCACerts; } catch (Exception e) { getLogger().error("Error during updating CA list", e); throw new CertificateException("Error during updating CA list", e, CERTIFICATE_ERROR); - } finally { - lock.unlock(); } } @@ -1013,42 +1031,37 @@ public boolean processCrl(CRLInfo crl) { return reinitCert; } - - private boolean removeCertificates(List certIds) { - lock.lock(); + private synchronized boolean removeCertificates(List certIds) { boolean reInitCert = false; - try { - // For now, remove self cert and ca cert is not implemented - // both requires a restart of the service. - if ((certSerialId != null && certIds.contains(certSerialId)) || - (caCertId != null && certIds.contains(caCertId)) || - (rootCaCertId != null && certIds.contains(rootCaCertId))) { - reInitCert = true; - } - Path basePath = securityConfig.getCertificateLocation(component); - for (String certId : certIds) { - if (certificateMap.containsKey(certId)) { - // remove on disk - String certName = String.format(CERT_FILE_NAME_FORMAT, certId); - - if (certId.equals(caCertId)) { - certName = CA_CERT_PREFIX + certName; - } + // For now, remove self cert and ca cert is not implemented + // both requires a restart of the service. + if ((certSerialId != null && certIds.contains(certSerialId)) || + (caCertId != null && certIds.contains(caCertId)) || + (rootCaCertId != null && certIds.contains(rootCaCertId))) { + reInitCert = true; + } - if (certId.equals(rootCaCertId)) { - certName = ROOT_CA_CERT_PREFIX + certName; - } + Path basePath = securityConfig.getCertificateLocation(component); + for (String certId : certIds) { + if (certificateMap.containsKey(certId)) { + // remove on disk + String certName = String.format(CERT_FILE_NAME_FORMAT, certId); - FileUtils.deleteQuietly(basePath.resolve(certName).toFile()); - // remove in memory - certificateMap.remove(certId); + if (certId.equals(caCertId)) { + certName = CA_CERT_PREFIX + certName; + } - // TODO: reset certSerialId, caCertId or rootCaCertId + if (certId.equals(rootCaCertId)) { + certName = ROOT_CA_CERT_PREFIX + certName; } + + FileUtils.deleteQuietly(basePath.resolve(certName).toFile()); + // remove in memory + certificateMap.remove(certId); + + // TODO: reset certSerialId, caCertId or rootCaCertId } - } finally { - lock.unlock(); } return reInitCert; } @@ -1087,6 +1100,10 @@ public KeyStoresFactory getClientKeyStoresFactory() @Override public synchronized void close() throws IOException { + if (executorService != null) { + executorService.shutdown(); + } + if (serverKeyStoresFactory != null) { serverKeyStoresFactory.destroy(); } @@ -1095,4 +1112,332 @@ public synchronized void close() throws IOException { clientKeyStoresFactory.destroy(); } } + + /** + * Check how much time before certificate will enter expiry grace period. + * @return Duration, time before certificate enters the grace + * period defined by "hdds.x509.renew.grace.duration" + */ + public Duration timeBeforeExpiryGracePeriod(X509Certificate certificate) { + Duration gracePeriod = securityConfig.getRenewalGracePeriod(); + Date expireDate = certificate.getNotAfter(); + LocalDateTime gracePeriodStart = expireDate.toInstant() + .atZone(ZoneId.systemDefault()).toLocalDateTime().minus(gracePeriod); + LocalDateTime currentTime = LocalDateTime.now(); + if (gracePeriodStart.isBefore(currentTime)) { + // Cert is already in grace period time. + return Duration.ZERO; + } else { + return Duration.between(currentTime, gracePeriodStart); + } + } + + /** + * Renew keys and certificate. Save the keys are certificate to disk in new + * directories, swap the current key directory and certs directory with the + * new directories. + * @param force, check certificate expiry time again if force is false. + * @return String, new certificate ID + * */ + public String renewAndStoreKeyAndCertificate(boolean force) + throws CertificateException { + if (!force) { + synchronized (this) { + Preconditions.checkArgument( + timeBeforeExpiryGracePeriod(x509Certificate).isZero()); + } + } + + String newKeyPath = securityConfig.getKeyLocation(component) + .toString() + HDDS_NEW_KEY_CERT_DIR_NAME_SUFFIX; + String newCertPath = securityConfig.getCertificateLocation(component) + .toString() + HDDS_NEW_KEY_CERT_DIR_NAME_SUFFIX; + File newKeyDir = new File(newKeyPath); + File newCertDir = new File(newCertPath); + try { + FileUtils.deleteDirectory(newKeyDir); + FileUtils.deleteDirectory(newCertDir); + Files.createDirectories(newKeyDir.toPath()); + Files.createDirectories(newCertDir.toPath()); + } catch (IOException e) { + throw new CertificateException("Error while deleting/creating " + + newKeyPath + " or " + newCertPath + " directories to cleanup " + + " certificate storage. ", e, RENEW_ERROR); + } + + // Generate key + KeyCodec newKeyCodec = new KeyCodec(securityConfig, newKeyDir.toPath()); + KeyPair newKeyPair; + try { + newKeyPair = createKeyPair(newKeyCodec); + } catch (CertificateException e) { + throw new CertificateException("Error while creating new key pair.", + e, RENEW_ERROR); + } + + // Get certificate signed + String newCertSerialId; + try { + CertificateSignRequest.Builder csrBuilder = getCSRBuilder(newKeyPair); + newCertSerialId = signAndStoreCertificate(csrBuilder.build(), + Paths.get(newCertPath)); + } catch (Exception e) { + throw new CertificateException("Error while signing and storing new" + + " certificates.", e, RENEW_ERROR); + } + + // switch Key and Certs directory on disk + File currentKeyDir = new File( + securityConfig.getKeyLocation(component).toString()); + File currentCertDir = new File( + securityConfig.getCertificateLocation(component).toString()); + File backupKeyDir = new File( + securityConfig.getKeyLocation(component).toString() + + HDDS_BACKUP_KEY_CERT_DIR_NAME_SUFFIX); + File backupCertDir = new File( + securityConfig.getCertificateLocation(component).toString() + + HDDS_BACKUP_KEY_CERT_DIR_NAME_SUFFIX); + + try { + Files.move(currentKeyDir.toPath(), backupKeyDir.toPath(), + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + // Cannot move current key dir to the backup dir + throw new CertificateException("Failed to move " + + currentKeyDir.getAbsolutePath() + + " to " + backupKeyDir.getAbsolutePath() + " during " + + "certificate renew.", RENEW_ERROR); + } + + try { + Files.move(currentCertDir.toPath(), backupCertDir.toPath(), + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + // Cannot move current cert dir to the backup dir + rollbackBackupDir(currentKeyDir, currentCertDir, backupKeyDir, + backupCertDir); + throw new CertificateException("Failed to move " + + currentCertDir.getAbsolutePath() + + " to " + backupCertDir.getAbsolutePath() + " during " + + "certificate renew.", RENEW_ERROR); + } + + try { + Files.move(newKeyDir.toPath(), currentKeyDir.toPath(), + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + // Cannot move new dir as the current dir + String msg = "Failed to move " + newKeyDir.getAbsolutePath() + + " to " + currentKeyDir.getAbsolutePath() + + " during certificate renew."; + // rollback + rollbackBackupDir(currentKeyDir, currentCertDir, backupKeyDir, + backupCertDir); + throw new CertificateException(msg, RENEW_ERROR); + } + + try { + Files.move(newCertDir.toPath(), currentCertDir.toPath(), + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + // Cannot move new dir as the current dir + String msg = "Failed to move " + newCertDir.getAbsolutePath() + + " to " + currentCertDir.getAbsolutePath() + + " during certificate renew."; + // delete currentKeyDir which is moved from new key directory + try { + FileUtils.deleteDirectory(new File(currentKeyDir.toString())); + } catch (IOException e1) { + getLogger().error("Failed to delete current KeyDir {} which is moved " + + " from the newly generated KeyDir {}", currentKeyDir, newKeyDir, e); + throw new CertificateException(msg, RENEW_ERROR); + } + // rollback + rollbackBackupDir(currentKeyDir, currentCertDir, backupKeyDir, + backupCertDir); + throw new CertificateException(msg, RENEW_ERROR); + } + + getLogger().info("Successful renew key and certificate." + + " New certificate {}.", newCertSerialId); + return newCertSerialId; + } + + private void rollbackBackupDir(File currentKeyDir, File currentCertDir, + File backupKeyDir, File backupCertDir) throws CertificateException { + // move backup dir back as current dir + try { + Files.move(backupKeyDir.toPath(), currentKeyDir.toPath(), + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + String msg = "Failed to move " + backupKeyDir.getAbsolutePath() + + " back to " + currentKeyDir.getAbsolutePath() + + " during rollback."; + // Need a manual recover process. + throw new CertificateException(msg, ROLLBACK_ERROR); + } + + try { + Files.move(backupCertDir.toPath(), currentCertDir.toPath(), + StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + String msg = "Failed to move " + backupCertDir.getAbsolutePath() + + " back to " + currentCertDir.getAbsolutePath() + + " during rollback."; + // Need a manual recover process. + throw new CertificateException(msg, ROLLBACK_ERROR); + } + + Preconditions.checkArgument(currentCertDir.exists()); + Preconditions.checkArgument(currentKeyDir.exists()); + } + + /** + * Delete old backup key and cert directory. + */ + public void cleanBackupDir() { + File backupKeyDir = new File( + securityConfig.getKeyLocation(component).toString() + + HDDS_BACKUP_KEY_CERT_DIR_NAME_SUFFIX); + File backupCertDir = new File( + securityConfig.getCertificateLocation(component).toString() + + HDDS_BACKUP_KEY_CERT_DIR_NAME_SUFFIX); + if (backupKeyDir.exists()) { + try { + FileUtils.deleteDirectory(backupKeyDir); + } catch (IOException e) { + getLogger().error("Error while deleting {} directories for " + + "certificate storage cleanup.", backupKeyDir, e); + } + } + if (backupCertDir.exists()) { + try { + FileUtils.deleteDirectory(backupCertDir); + } catch (IOException e) { + getLogger().error("Error while deleting {} directories for " + + "certificate storage cleanup.", backupCertDir, e); + } + } + } + + synchronized void reloadKeyAndCertificate(String newCertId) { + // reset current value + privateKey = null; + publicKey = null; + x509Certificate = null; + certSerialId = null; + caCertId = null; + rootCaCertId = null; + + setCertificateId(newCertId); + getLogger().info("Reset and reload key and all certificates."); + } + + public SecurityConfig getSecurityConfig() { + return securityConfig; + } + + public OzoneConfiguration getConfig() { + return (OzoneConfiguration)securityConfig.getConfiguration(); + } + + @Override + public abstract String signAndStoreCertificate( + PKCS10CertificationRequest request, Path certPath) + throws CertificateException; + + public String signAndStoreCertificate(PKCS10CertificationRequest request) + throws CertificateException { + return signAndStoreCertificate(request, + getSecurityConfig().getCertificateLocation(getComponentName())); + } + + @Override + public abstract CertificateSignRequest.Builder getCSRBuilder(KeyPair keyPair) + throws CertificateException; + + public SCMSecurityProtocolClientSideTranslatorPB getScmSecureClient() + throws IOException { + if (scmSecurityProtocolClient == null) { + scmSecurityProtocolClient = + getScmSecurityClientWithMaxRetry( + (OzoneConfiguration) securityConfig.getConfiguration()); + } + return scmSecurityProtocolClient; + } + + @VisibleForTesting + public void setSecureScmClient( + SCMSecurityProtocolClientSideTranslatorPB client) { + scmSecurityProtocolClient = client; + } + + public synchronized void startCertificateMonitor() { + Preconditions.checkNotNull(getCertificate(), + "Component certificate should not be empty"); + // Schedule task to refresh certificate before it expires + Duration gracePeriod = securityConfig.getRenewalGracePeriod(); + long timeBeforeGracePeriod = + timeBeforeExpiryGracePeriod(x509Certificate).toMillis(); + // At least three chances to renew the certificate before it expires + long interval = + Math.min(gracePeriod.toMillis() / 3, TimeUnit.DAYS.toMillis(1)); + + if (executorService == null) { + executorService = Executors.newScheduledThreadPool(1, + new ThreadFactoryBuilder().setNameFormat("CertificateLifetimeMonitor") + .setDaemon(true).build()); + } + this.executorService.scheduleAtFixedRate(new CertificateLifetimeMonitor(), + timeBeforeGracePeriod, interval, TimeUnit.MILLISECONDS); + getLogger().info("CertificateLifetimeMonitor is started with first delay" + + " {} ms and interval {} ms.", timeBeforeGracePeriod, interval); + } + + /** + * Task to monitor certificate lifetime and renew the certificate if needed. + */ + public class CertificateLifetimeMonitor implements Runnable { + @Override + public void run() { + + renewLock.lock(); + try { + Duration timeLeft = timeBeforeExpiryGracePeriod(getCertificate()); + if (timeLeft.isZero()) { + String newCertId; + try { + getLogger().info("Current certificate has entered the expiry" + + " grace period {}. Starting renew key and certs.", + timeLeft, securityConfig.getRenewalGracePeriod()); + newCertId = renewAndStoreKeyAndCertificate(false); + } catch (CertificateException e) { + if (e.errorCode() == + CertificateException.ErrorCode.ROLLBACK_ERROR) { + if (shutdownCallback != null) { + getLogger().error("Failed to rollback key and cert after an " + + " unsuccessful renew try.", e); + shutdownCallback.run(); + } + } + getLogger().error("Failed to renew and store key and cert." + + " Keep using existing certificates.", e); + return; + } + + // Persist new cert serial id in component VERSION file + if (certIdSaveCallback != null) { + certIdSaveCallback.accept(newCertId); + } + + // reset and reload all certs + reloadKeyAndCertificate(newCertId); + // cleanup backup directory + cleanBackupDir(); + } + } finally { + renewLock.unlock(); + } + } + } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java deleted file mode 100644 index d6c535a98bd0..000000000000 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/OMCertificateClient.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.hadoop.hdds.security.x509.certificate.client; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hadoop.hdds.security.x509.SecurityConfig; - -/** - * Certificate client for OzoneManager. - */ -public class OMCertificateClient extends CommonCertificateClient { - - private static final Logger LOG = - LoggerFactory.getLogger(OMCertificateClient.class); - - public static final String COMPONENT_NAME = "om"; - - public OMCertificateClient(SecurityConfig securityConfig, - String certSerialId, String localCrlId) { - super(securityConfig, LOG, certSerialId, COMPONENT_NAME); - this.setLocalCrlId(localCrlId != null ? - Long.parseLong(localCrlId) : 0); - } - - public OMCertificateClient(SecurityConfig securityConfig, - String certSerialId) { - this(securityConfig, certSerialId, null); - } - - public OMCertificateClient(SecurityConfig securityConfig) { - this(securityConfig, null, null); - } - - @Override - public Logger getLogger() { - return LOG; - } -} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java index afbcbf643559..3140444c42f4 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/ReconCertificateClient.java @@ -17,10 +17,27 @@ */ package org.apache.hadoop.hdds.security.x509.certificate.client; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos; import org.apache.hadoop.hdds.security.x509.SecurityConfig; +import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; +import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; +import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; +import org.apache.hadoop.security.UserGroupInformation; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.net.InetAddress; +import java.nio.file.Path; +import java.security.KeyPair; +import java.util.function.Consumer; + +import static org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec.getX509Certificate; +import static org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString; +import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateException.ErrorCode.CSR_ERROR; + /** * Certificate client for Recon. */ @@ -29,10 +46,93 @@ public class ReconCertificateClient extends CommonCertificateClient { LoggerFactory.getLogger(ReconCertificateClient.class); public static final String COMPONENT_NAME = "recon"; + private final String clusterID; + private final String reconID; + + public ReconCertificateClient(SecurityConfig securityConfig, + String certSerialId, String clusterId, String reconId, + Consumer saveCertIdCallback, Runnable shutdownCallback) { + super(securityConfig, LOG, certSerialId, COMPONENT_NAME, + saveCertIdCallback, shutdownCallback); + this.clusterID = clusterId; + this.reconID = reconId; + } public ReconCertificateClient(SecurityConfig securityConfig, - String certSerialId) { - super(securityConfig, LOG, certSerialId, COMPONENT_NAME); + String certSerialId, String clusterId, String reconId) { + super(securityConfig, LOG, certSerialId, COMPONENT_NAME, null, null); + this.clusterID = clusterId; + this.reconID = reconId; + } + + @Override + public CertificateSignRequest.Builder getCSRBuilder() + throws CertificateException { + return getCSRBuilder(new KeyPair(getPublicKey(), getPrivateKey())); + } + + @Override + public CertificateSignRequest.Builder getCSRBuilder(KeyPair keyPair) + throws CertificateException { + LOG.info("Creating CSR for Recon."); + try { + CertificateSignRequest.Builder builder = super.getCSRBuilder(); + String hostname = InetAddress.getLocalHost().getCanonicalHostName(); + String subject = UserGroupInformation.getCurrentUser() + .getShortUserName() + "@" + hostname; + + builder.setCA(false) + .setKey(keyPair) + .setConfiguration(getConfig()) + .setSubject(subject); + + return builder; + } catch (Exception e) { + LOG.error("Failed to get hostname or current user", e); + throw new CertificateException("Failed to get hostname or current user", + e, CSR_ERROR); + } + } + + @Override + public String signAndStoreCertificate(PKCS10CertificationRequest csr, + Path certPath) throws CertificateException { + try { + SCMSecurityProtocolProtos.SCMGetCertResponseProto response; + HddsProtos.NodeDetailsProto.Builder reconDetailsProtoBuilder = + HddsProtos.NodeDetailsProto.newBuilder() + .setHostName(InetAddress.getLocalHost().getHostName()) + .setClusterId(clusterID) + .setUuid(reconID) + .setNodeType(HddsProtos.NodeType.RECON); + // TODO: For SCM CA we should fetch certificate from multiple SCMs. + response = getScmSecureClient().getCertificateChain( + reconDetailsProtoBuilder.build(), getEncodedString(csr)); + + // Persist certificates. + if (response.hasX509CACertificate()) { + String pemEncodedCert = response.getX509Certificate(); + CertificateCodec certCodec = new CertificateCodec( + getSecurityConfig(), certPath); + storeCertificate(pemEncodedCert, true, false, false, certCodec, false); + storeCertificate(response.getX509CACertificate(), true, true, + false, certCodec, false); + + // Store Root CA certificate. + if (response.hasX509RootCACertificate()) { + storeCertificate(response.getX509RootCACertificate(), + true, false, true, certCodec, false); + } + return getX509Certificate(pemEncodedCert).getSerialNumber().toString(); + } else { + throw new CertificateException("Unable to retrieve recon certificate " + + "chain"); + } + } catch (IOException | java.security.cert.CertificateException e) { + LOG.error("Error while signing and storing SCM signed certificate.", e); + throw new CertificateException( + "Error while signing and storing SCM signed certificate.", e); + } } @Override diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java index 91acc1e767a0..242ffaca8e85 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/SCMCertificateClient.java @@ -22,10 +22,13 @@ import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; import org.apache.hadoop.ozone.OzoneConsts; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.file.Path; import java.nio.file.Paths; +import java.security.KeyPair; import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.FAILURE; import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.GETCERT; @@ -48,16 +51,16 @@ public class SCMCertificateClient extends DefaultCertificateClient { public SCMCertificateClient(SecurityConfig securityConfig, String certSerialId) { - super(securityConfig, LOG, certSerialId, COMPONENT_NAME); + super(securityConfig, LOG, certSerialId, COMPONENT_NAME, null, null); } public SCMCertificateClient(SecurityConfig securityConfig) { - super(securityConfig, LOG, null, COMPONENT_NAME); + super(securityConfig, LOG, null, COMPONENT_NAME, null, null); } public SCMCertificateClient(SecurityConfig securityConfig, String certSerialId, String component) { - super(securityConfig, LOG, certSerialId, component); + super(securityConfig, LOG, certSerialId, component, null, null); } @Override @@ -140,4 +143,16 @@ public CertificateSignRequest.Builder getCSRBuilder() public Logger getLogger() { return LOG; } + + @Override + public String signAndStoreCertificate(PKCS10CertificationRequest request, + Path certPath) throws CertificateException { + return null; + } + + @Override + public CertificateSignRequest.Builder getCSRBuilder(KeyPair keyPair) + throws CertificateException { + return null; + } } \ No newline at end of file diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java index b3121283b18e..89fde76cb38f 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/exceptions/CertificateException.java @@ -74,6 +74,10 @@ public CertificateException(Throwable cause) { super(cause); } + public ErrorCode errorCode() { + return errorCode; + } + /** * Error codes to make it easy to decode these exceptions. */ @@ -84,6 +88,8 @@ public enum ErrorCode { BOOTSTRAP_ERROR, CSR_ERROR, CRYPTO_SIGNATURE_VERIFICATION_ERROR, - CERTIFICATE_NOT_FOUND_ERROR + CERTIFICATE_NOT_FOUND_ERROR, + RENEW_ERROR, + ROLLBACK_ERROR } } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java index aead3582249e..ab175d7838b9 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java @@ -89,6 +89,18 @@ public KeyCodec(SecurityConfig config, String component) { this.location = securityConfig.getKeyLocation(component); } + /** + * Creates a KeyCodec with component name. + * + * @param config - Security Config. + * @param keyDir - path to save the key materials. + */ + public KeyCodec(SecurityConfig config, Path keyDir) { + this.securityConfig = config; + isPosixFileSystem = KeyCodec::isPosix; + this.location = keyDir; + } + /** * Checks if File System supports posix style security permissions. * diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java index 63a34ef2369d..ad435a0f30b5 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; -import org.apache.hadoop.hdds.security.x509.certificate.client.OMCertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.token.Token; import org.apache.ozone.test.GenericTestUtils; @@ -106,7 +106,7 @@ public void setUp() throws Exception { omCertSerialId = x509Certificate.getSerialNumber().toString(); secretManager = new OzoneBlockTokenSecretManager(securityConfig, TimeUnit.HOURS.toMillis(1), omCertSerialId); - client = Mockito.mock(OMCertificateClient.class); + client = Mockito.mock(DefaultCertificateClient.class); when(client.getCertificate()).thenReturn(x509Certificate); when(client.getCertificate(anyString())). thenReturn(x509Certificate); diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java index f1bbaff47b9a..1c5e1118e0bb 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.file.Path; import java.security.KeyPair; import java.security.PrivateKey; import java.security.PublicKey; @@ -34,6 +35,7 @@ import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; /** * Test implementation for CertificateClient. To be used only for test @@ -43,7 +45,6 @@ public class CertificateClientTest implements CertificateClient { private KeyPair keyPair; private X509Certificate x509Certificate; - private boolean isKeyRenewed; private SecurityConfig secConfig; public CertificateClientTest(OzoneConfiguration conf) @@ -91,6 +92,10 @@ public boolean verifyCertificate(X509Certificate certificate) { return true; } + @Override + public void setCertificateId(String certSerialId) { + } + @Override public byte[] signDataStream(InputStream stream) throws CertificateException { @@ -114,11 +119,29 @@ public boolean verifySignature(byte[] data, byte[] signature, return true; } + @Override + public CertificateSignRequest.Builder getCSRBuilder(KeyPair key) + throws IOException { + return null; + } + @Override public CertificateSignRequest.Builder getCSRBuilder() { return new CertificateSignRequest.Builder(); } + @Override + public String signAndStoreCertificate(PKCS10CertificationRequest request, + Path certPath) throws CertificateException { + return null; + } + + @Override + public String signAndStoreCertificate(PKCS10CertificationRequest request) + throws CertificateException { + return null; + } + @Override public X509Certificate queryCertificate(String query) { return null; @@ -230,11 +253,6 @@ public KeyStoresFactory getClientKeyStoresFactory() return null; } - @Override - public boolean isCertificateRenewed() { - return isKeyRenewed; - } - public void renewKey() throws Exception { KeyPair newKeyPair = KeyStoreTestUtil.generateKeyPair("RSA"); X509Certificate newCert = KeyStoreTestUtil.generateCertificate( @@ -242,7 +260,6 @@ public void renewKey() throws Exception { keyPair = newKeyPair; x509Certificate = newCert; - isKeyRenewed = true; } @Override diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDefaultCertificateClient.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDefaultCertificateClient.java index 394f4b0a3b75..0e7beea5eb20 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDefaultCertificateClient.java @@ -18,15 +18,23 @@ */ package org.apache.hadoop.hdds.security.x509.certificate.client; +import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos; +import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; +import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; import org.apache.hadoop.hdds.security.x509.keys.KeyCodec; import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; +import org.junit.Assert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -66,6 +74,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyObject; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.mock; @@ -79,16 +88,11 @@ public class TestDefaultCertificateClient { private String certSerialId; private X509Certificate x509Certificate; - private OMCertificateClient omCertClient; private DNCertificateClient dnCertClient; private HDDSKeyGenerator keyGenerator; - private Path omMetaDirPath; private Path dnMetaDirPath; - private SecurityConfig omSecurityConfig; private SecurityConfig dnSecurityConfig; private static final String DN_COMPONENT = DNCertificateClient.COMPONENT_NAME; - private static final String OM_COMPONENT = OMCertificateClient.COMPONENT_NAME; - private KeyCodec omKeyCodec; private KeyCodec dnKeyCodec; @BeforeEach @@ -96,25 +100,16 @@ public void setUp() throws Exception { OzoneConfiguration config = new OzoneConfiguration(); config.setStrings(OZONE_SCM_NAMES, "localhost"); config.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 2); - final String omPath = GenericTestUtils - .getTempPath(UUID.randomUUID().toString()); final String dnPath = GenericTestUtils .getTempPath(UUID.randomUUID().toString()); - omMetaDirPath = Paths.get(omPath, "test"); dnMetaDirPath = Paths.get(dnPath, "test"); - - config.set(HDDS_METADATA_DIR_NAME, omMetaDirPath.toString()); - omSecurityConfig = new SecurityConfig(config); config.set(HDDS_METADATA_DIR_NAME, dnMetaDirPath.toString()); dnSecurityConfig = new SecurityConfig(config); - - keyGenerator = new HDDSKeyGenerator(omSecurityConfig); - omKeyCodec = new KeyCodec(omSecurityConfig, OM_COMPONENT); + keyGenerator = new HDDSKeyGenerator(dnSecurityConfig); dnKeyCodec = new KeyCodec(dnSecurityConfig, DN_COMPONENT); - Files.createDirectories(omSecurityConfig.getKeyLocation(OM_COMPONENT)); Files.createDirectories(dnSecurityConfig.getKeyLocation(DN_COMPONENT)); x509Certificate = generateX509Cert(null); certSerialId = x509Certificate.getSerialNumber().toString(); @@ -122,15 +117,14 @@ public void setUp() throws Exception { } private void getCertClient() { - omCertClient = new OMCertificateClient(omSecurityConfig, certSerialId); - dnCertClient = new DNCertificateClient(dnSecurityConfig, certSerialId); + dnCertClient = new DNCertificateClient(dnSecurityConfig, + MockDatanodeDetails.randomDatanodeDetails(), certSerialId, null, + () -> System.exit(1)); } @AfterEach public void tearDown() { - omCertClient = null; dnCertClient = null; - FileUtils.deleteQuietly(omMetaDirPath.toFile()); FileUtils.deleteQuietly(dnMetaDirPath.toFile()); } @@ -141,13 +135,13 @@ public void tearDown() { @Test public void testKeyOperations() throws Exception { cleanupOldKeyPair(); - PrivateKey pvtKey = omCertClient.getPrivateKey(); - PublicKey publicKey = omCertClient.getPublicKey(); + PrivateKey pvtKey = dnCertClient.getPrivateKey(); + PublicKey publicKey = dnCertClient.getPublicKey(); assertNull(publicKey); assertNull(pvtKey); KeyPair keyPair = generateKeyPairFiles(); - pvtKey = omCertClient.getPrivateKey(); + pvtKey = dnCertClient.getPrivateKey(); assertNotNull(pvtKey); assertEquals(pvtKey, keyPair.getPrivate()); @@ -159,21 +153,12 @@ public void testKeyOperations() throws Exception { private KeyPair generateKeyPairFiles() throws Exception { cleanupOldKeyPair(); KeyPair keyPair = keyGenerator.generateKey(); - omKeyCodec.writePrivateKey(keyPair.getPrivate()); - omKeyCodec.writePublicKey(keyPair.getPublic()); - dnKeyCodec.writePrivateKey(keyPair.getPrivate()); dnKeyCodec.writePublicKey(keyPair.getPublic()); return keyPair; } private void cleanupOldKeyPair() { - FileUtils.deleteQuietly(Paths.get( - omSecurityConfig.getKeyLocation(OM_COMPONENT).toString(), - omSecurityConfig.getPrivateKeyFileName()).toFile()); - FileUtils.deleteQuietly(Paths.get( - omSecurityConfig.getKeyLocation(OM_COMPONENT).toString(), - omSecurityConfig.getPublicKeyFileName()).toFile()); FileUtils.deleteQuietly(Paths.get( dnSecurityConfig.getKeyLocation(DN_COMPONENT).toString(), dnSecurityConfig.getPrivateKeyFileName()).toFile()); @@ -187,12 +172,12 @@ private void cleanupOldKeyPair() { */ @Test public void testCertificateOps() throws Exception { - X509Certificate cert = omCertClient.getCertificate(); + X509Certificate cert = dnCertClient.getCertificate(); assertNull(cert); - omCertClient.storeCertificate(getPEMEncodedString(x509Certificate), + dnCertClient.storeCertificate(getPEMEncodedString(x509Certificate), true); - cert = omCertClient.getCertificate( + cert = dnCertClient.getCertificate( x509Certificate.getSerialNumber().toString()); assertNotNull(cert); assertTrue(cert.getEncoded().length > 0); @@ -206,26 +191,26 @@ private X509Certificate generateX509Cert(KeyPair keyPair) throws Exception { keyPair = generateKeyPairFiles(); } return KeyStoreTestUtil.generateCertificate("CN=Test", keyPair, 30, - omSecurityConfig.getSignatureAlgo()); + dnSecurityConfig.getSignatureAlgo()); } @Test public void testSignDataStream() throws Exception { String data = RandomStringUtils.random(100); FileUtils.deleteQuietly(Paths.get( - omSecurityConfig.getKeyLocation(OM_COMPONENT).toString(), - omSecurityConfig.getPrivateKeyFileName()).toFile()); + dnSecurityConfig.getKeyLocation(DN_COMPONENT).toString(), + dnSecurityConfig.getPrivateKeyFileName()).toFile()); FileUtils.deleteQuietly(Paths.get( - omSecurityConfig.getKeyLocation(OM_COMPONENT).toString(), - omSecurityConfig.getPublicKeyFileName()).toFile()); + dnSecurityConfig.getKeyLocation(DN_COMPONENT).toString(), + dnSecurityConfig.getPublicKeyFileName()).toFile()); // Expect error when there is no private key to sign. LambdaTestUtils.intercept(IOException.class, "Error while " + "signing the stream", - () -> omCertClient.signDataStream(IOUtils.toInputStream(data, UTF_8))); + () -> dnCertClient.signDataStream(IOUtils.toInputStream(data, UTF_8))); generateKeyPairFiles(); - byte[] sign = omCertClient.signDataStream(IOUtils.toInputStream(data, + byte[] sign = dnCertClient.signDataStream(IOUtils.toInputStream(data, UTF_8)); validateHash(sign, data.getBytes(UTF_8)); } @@ -236,9 +221,9 @@ public void testSignDataStream() throws Exception { private void validateHash(byte[] hash, byte[] data) throws Exception { Signature rsaSignature = - Signature.getInstance(omSecurityConfig.getSignatureAlgo(), - omSecurityConfig.getProvider()); - rsaSignature.initVerify(omCertClient.getPublicKey()); + Signature.getInstance(dnSecurityConfig.getSignatureAlgo(), + dnSecurityConfig.getProvider()); + rsaSignature.initVerify(dnCertClient.getPublicKey()); rsaSignature.update(data); assertTrue(rsaSignature.verify(hash)); } @@ -249,20 +234,20 @@ private void validateHash(byte[] hash, byte[] data) @Test public void verifySignatureStream() throws Exception { String data = RandomStringUtils.random(500); - byte[] sign = omCertClient.signDataStream(IOUtils.toInputStream(data, + byte[] sign = dnCertClient.signDataStream(IOUtils.toInputStream(data, UTF_8)); // Positive tests. - assertTrue(omCertClient.verifySignature(data.getBytes(UTF_8), sign, + assertTrue(dnCertClient.verifySignature(data.getBytes(UTF_8), sign, x509Certificate)); - assertTrue(omCertClient.verifySignature( + assertTrue(dnCertClient.verifySignature( IOUtils.toInputStream(data, UTF_8), sign, x509Certificate)); // Negative tests. - assertFalse(omCertClient.verifySignature(data.getBytes(UTF_8), + assertFalse(dnCertClient.verifySignature(data.getBytes(UTF_8), "abc".getBytes(UTF_8), x509Certificate)); - assertFalse(omCertClient.verifySignature(IOUtils.toInputStream(data, + assertFalse(dnCertClient.verifySignature(IOUtils.toInputStream(data, UTF_8), "abc".getBytes(UTF_8), x509Certificate)); } @@ -273,19 +258,19 @@ public void verifySignatureStream() throws Exception { @Test public void verifySignatureDataArray() throws Exception { String data = RandomStringUtils.random(500); - byte[] sign = omCertClient.signData(data.getBytes(UTF_8)); + byte[] sign = dnCertClient.signData(data.getBytes(UTF_8)); // Positive tests. - assertTrue(omCertClient.verifySignature(data.getBytes(UTF_8), sign, + assertTrue(dnCertClient.verifySignature(data.getBytes(UTF_8), sign, x509Certificate)); - assertTrue(omCertClient.verifySignature( + assertTrue(dnCertClient.verifySignature( IOUtils.toInputStream(data, UTF_8), sign, x509Certificate)); // Negative tests. - assertFalse(omCertClient.verifySignature(data.getBytes(UTF_8), + assertFalse(dnCertClient.verifySignature(data.getBytes(UTF_8), "abc".getBytes(UTF_8), x509Certificate)); - assertFalse(omCertClient.verifySignature(IOUtils.toInputStream(data, + assertFalse(dnCertClient.verifySignature(IOUtils.toInputStream(data, UTF_8), "abc".getBytes(UTF_8), x509Certificate)); } @@ -294,7 +279,7 @@ public void verifySignatureDataArray() throws Exception { public void queryCertificate() throws Exception { LambdaTestUtils.intercept(UnsupportedOperationException.class, "Operation not supported", - () -> omCertClient.queryCertificate("")); + () -> dnCertClient.queryCertificate("")); } @Test @@ -329,7 +314,8 @@ public void testCertificateLoadingOnInit() throws Exception { getPEMEncodedString(cert3), true); // Re instantiate DN client which will load certificates from filesystem. - dnCertClient = new DNCertificateClient(dnSecurityConfig, certSerialId); + dnCertClient = new DNCertificateClient(dnSecurityConfig, null, + certSerialId, null, null); assertNotNull(dnCertClient.getCertificate(cert1.getSerialNumber() .toString())); @@ -361,66 +347,34 @@ public void testStoreCertificate() throws Exception { @Test public void testInitCertAndKeypairValidationFailures() throws Exception { - GenericTestUtils.LogCapturer dnClientLog = GenericTestUtils.LogCapturer .captureLogs(dnCertClient.getLogger()); - GenericTestUtils.LogCapturer omClientLog = GenericTestUtils.LogCapturer - .captureLogs(omCertClient.getLogger()); KeyPair keyPair = keyGenerator.generateKey(); - KeyPair keyPair2 = keyGenerator.generateKey(); + KeyPair keyPair1 = keyGenerator.generateKey(); dnClientLog.clearOutput(); - omClientLog.clearOutput(); // Case 1. Expect failure when keypair validation fails. - FileUtils.deleteQuietly(Paths.get( - omSecurityConfig.getKeyLocation(OM_COMPONENT).toString(), - omSecurityConfig.getPrivateKeyFileName()).toFile()); - FileUtils.deleteQuietly(Paths.get( - omSecurityConfig.getKeyLocation(OM_COMPONENT).toString(), - omSecurityConfig.getPublicKeyFileName()).toFile()); - - FileUtils.deleteQuietly(Paths.get( dnSecurityConfig.getKeyLocation(DN_COMPONENT).toString(), dnSecurityConfig.getPrivateKeyFileName()).toFile()); FileUtils.deleteQuietly(Paths.get( dnSecurityConfig.getKeyLocation(DN_COMPONENT).toString(), dnSecurityConfig.getPublicKeyFileName()).toFile()); - - omKeyCodec.writePrivateKey(keyPair.getPrivate()); - omKeyCodec.writePublicKey(keyPair2.getPublic()); - dnKeyCodec.writePrivateKey(keyPair.getPrivate()); - dnKeyCodec.writePublicKey(keyPair2.getPublic()); - + dnKeyCodec.writePublicKey(keyPair1.getPublic()); // Check for DN. assertEquals(FAILURE, dnCertClient.init()); assertTrue(dnClientLog.getOutput().contains("Keypair validation failed")); dnClientLog.clearOutput(); - omClientLog.clearOutput(); - - // Check for OM. - assertEquals(FAILURE, omCertClient.init()); - assertTrue(omClientLog.getOutput().contains("Keypair validation failed")); - dnClientLog.clearOutput(); - omClientLog.clearOutput(); // Case 2. Expect failure when certificate is generated from different // private key and keypair validation fails. getCertClient(); - FileUtils.deleteQuietly(Paths.get( - omSecurityConfig.getKeyLocation(OM_COMPONENT).toString(), - omSecurityConfig.getCertificateFileName()).toFile()); FileUtils.deleteQuietly(Paths.get( dnSecurityConfig.getKeyLocation(DN_COMPONENT).toString(), dnSecurityConfig.getCertificateFileName()).toFile()); - CertificateCodec omCertCodec = new CertificateCodec(omSecurityConfig, - OM_COMPONENT); - omCertCodec.writeCertificate(new X509CertificateHolder( - x509Certificate.getEncoded())); - CertificateCodec dnCertCodec = new CertificateCodec(dnSecurityConfig, DN_COMPONENT); dnCertCodec.writeCertificate(new X509CertificateHolder( @@ -429,26 +383,15 @@ public void testInitCertAndKeypairValidationFailures() throws Exception { assertEquals(FAILURE, dnCertClient.init()); assertTrue(dnClientLog.getOutput().contains("Keypair validation failed")); dnClientLog.clearOutput(); - omClientLog.clearOutput(); - - // Check for OM. - assertEquals(FAILURE, omCertClient.init()); - assertTrue(omClientLog.getOutput().contains("Keypair validation failed")); - dnClientLog.clearOutput(); - omClientLog.clearOutput(); // Case 3. Expect failure when certificate is generated from different // private key and certificate validation fails. // Re-write the correct public key. - FileUtils.deleteQuietly(Paths.get( - omSecurityConfig.getKeyLocation(OM_COMPONENT).toString(), - omSecurityConfig.getPublicKeyFileName()).toFile()); FileUtils.deleteQuietly(Paths.get( dnSecurityConfig.getKeyLocation(DN_COMPONENT).toString(), dnSecurityConfig.getPublicKeyFileName()).toFile()); getCertClient(); - omKeyCodec.writePublicKey(keyPair.getPublic()); dnKeyCodec.writePublicKey(keyPair.getPublic()); // Check for DN. @@ -456,20 +399,9 @@ public void testInitCertAndKeypairValidationFailures() throws Exception { assertTrue(dnClientLog.getOutput() .contains("Stored certificate is generated with different")); dnClientLog.clearOutput(); - omClientLog.clearOutput(); - - //Check for OM. - assertEquals(FAILURE, omCertClient.init()); - assertTrue(omClientLog.getOutput() - .contains("Stored certificate is generated with different")); - dnClientLog.clearOutput(); - omClientLog.clearOutput(); // Case 4. Failure when public key recovery fails. getCertClient(); - FileUtils.deleteQuietly(Paths.get( - omSecurityConfig.getKeyLocation(OM_COMPONENT).toString(), - omSecurityConfig.getPublicKeyFileName()).toFile()); FileUtils.deleteQuietly(Paths.get( dnSecurityConfig.getKeyLocation(DN_COMPONENT).toString(), dnSecurityConfig.getPublicKeyFileName()).toFile()); @@ -477,12 +409,6 @@ public void testInitCertAndKeypairValidationFailures() throws Exception { // Check for DN. assertEquals(FAILURE, dnCertClient.init()); assertTrue(dnClientLog.getOutput().contains("Can't recover public key")); - - // Check for OM. - assertEquals(FAILURE, omCertClient.init()); - assertTrue(omClientLog.getOutput().contains("Can't recover public key")); - dnClientLog.clearOutput(); - omClientLog.clearOutput(); } @Test @@ -505,7 +431,8 @@ public void testCertificateExpirationHandlingInInit() throws Exception { when(mockCert.getNotAfter()).thenReturn(expiration); DefaultCertificateClient client = - new DefaultCertificateClient(config, mockLogger, certId, compName) { + new DefaultCertificateClient(config, mockLogger, certId, compName, + null, null) { @Override public PrivateKey getPrivateKey() { return mock(PrivateKey.class); @@ -520,10 +447,113 @@ public PublicKey getPublicKey() { public X509Certificate getCertificate() { return mockCert; } + + @Override + public String signAndStoreCertificate( + PKCS10CertificationRequest request, Path certPath) + throws CertificateException { + return null; + } + + @Override + public CertificateSignRequest.Builder getCSRBuilder(KeyPair keyPair) + throws CertificateException { + return null; + } }; InitResponse resp = client.init(); verify(mockLogger, atLeastOnce()).info(anyString()); assertEquals(resp, REINIT); } + + @Test + public void testTimeBeforeExpiryGracePeriod() throws Exception { + KeyPair keyPair = keyGenerator.generateKey(); + Duration gracePeriod = dnSecurityConfig.getRenewalGracePeriod(); + + X509Certificate cert = KeyStoreTestUtil.generateCertificate("CN=Test", + keyPair, (int)(gracePeriod.toDays()), + dnSecurityConfig.getSignatureAlgo()); + dnCertClient.storeCertificate(getPEMEncodedString(cert), true); + Duration duration = dnCertClient.timeBeforeExpiryGracePeriod(cert); + Assert.assertTrue(duration.isZero()); + + cert = KeyStoreTestUtil.generateCertificate("CN=Test", + keyPair, (int)(gracePeriod.toDays() + 1), + dnSecurityConfig.getSignatureAlgo()); + dnCertClient.storeCertificate(getPEMEncodedString(cert), true); + duration = dnCertClient.timeBeforeExpiryGracePeriod(cert); + Assert.assertTrue(duration.toMillis() < Duration.ofDays(1).toMillis() && + duration.toMillis() > Duration.ofHours(23).plusMinutes(59).toMillis()); + } + + @Test + public void testRenewAndStoreKeyAndCertificate() throws Exception { + // save the certificate on dn + CertificateCodec certCodec = new CertificateCodec(dnSecurityConfig, + dnSecurityConfig.getCertificateLocation(DN_COMPONENT)); + certCodec.writeCertificate( + new X509CertificateHolder(x509Certificate.getEncoded())); + + SCMSecurityProtocolClientSideTranslatorPB scmClient = + mock(SCMSecurityProtocolClientSideTranslatorPB.class); + X509Certificate newCert = generateX509Cert(null); + dnCertClient.setSecureScmClient(scmClient); + String pemCert = CertificateCodec.getPEMEncodedString(newCert); + SCMSecurityProtocolProtos.SCMGetCertResponseProto responseProto = + SCMSecurityProtocolProtos.SCMGetCertResponseProto + .newBuilder().setResponseCode(SCMSecurityProtocolProtos + .SCMGetCertResponseProto.ResponseCode.success) + .setX509Certificate(pemCert) + .setX509CACertificate(pemCert) + .build(); + when(scmClient.getDataNodeCertificateChain(anyObject(), anyString())) + .thenReturn(responseProto); + + String certID = dnCertClient.getCertificate().getSerialNumber().toString(); + // a success renew + String newCertId = dnCertClient.renewAndStoreKeyAndCertificate(true); + Assert.assertFalse(certID.equals(newCertId)); + Assert.assertTrue(dnCertClient.getCertificate().getSerialNumber() + .toString().equals(certID)); + + File newKeyDir = new File(dnSecurityConfig.getKeyLocation( + dnCertClient.getComponentName()).toString() + + HddsConfigKeys.HDDS_NEW_KEY_CERT_DIR_NAME_SUFFIX); + File newCertDir = new File(dnSecurityConfig.getCertificateLocation( + dnCertClient.getComponentName()).toString() + + HddsConfigKeys.HDDS_NEW_KEY_CERT_DIR_NAME_SUFFIX); + File backupKeyDir = new File(dnSecurityConfig.getKeyLocation( + dnCertClient.getComponentName()).toString() + + HddsConfigKeys.HDDS_BACKUP_KEY_CERT_DIR_NAME_SUFFIX); + File backupCertDir = new File(dnSecurityConfig.getCertificateLocation( + dnCertClient.getComponentName()).toString() + + HddsConfigKeys.HDDS_BACKUP_KEY_CERT_DIR_NAME_SUFFIX); + + // backup directories exist + Assert.assertTrue(backupKeyDir.exists()); + Assert.assertTrue(backupCertDir.exists()); + // new directories should not exist + Assert.assertFalse(newKeyDir.exists()); + Assert.assertFalse(newCertDir.exists()); + + // cleanup backup key and cert dir + dnCertClient.cleanBackupDir(); + + Files.createDirectories(newKeyDir.toPath()); + Files.createDirectories(newCertDir.toPath()); + KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA"); + KeyCodec newKeyCodec = new KeyCodec(dnSecurityConfig, newKeyDir.toPath()); + newKeyCodec.writeKey(keyPair); + + X509Certificate cert = KeyStoreTestUtil.generateCertificate( + "CN=OzoneMaster", keyPair, 30, "SHA256withRSA"); + certCodec = new CertificateCodec(dnSecurityConfig, + newCertDir.toPath()); + dnCertClient.storeCertificate(getPEMEncodedString(cert), true, false, false, + certCodec, false); + // a success renew after auto cleanup new key and cert dir + dnCertClient.renewAndStoreKeyAndCertificate(true); + } } \ No newline at end of file diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDnCertificateClientInit.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDnCertificateClientInit.java new file mode 100644 index 000000000000..9b31426fce69 --- /dev/null +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestDnCertificateClientInit.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.hadoop.hdds.security.x509.certificate.client; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.security.x509.SecurityConfig; +import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; +import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator; +import org.apache.hadoop.hdds.security.x509.keys.KeyCodec; +import org.apache.hadoop.ozone.OzoneSecurityUtil; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.ozone.test.GenericTestUtils; +import org.bouncycastle.cert.X509CertificateHolder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME; +import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse; +import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.FAILURE; +import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.GETCERT; +import static org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient.InitResponse.SUCCESS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +/** + * Test class for {@link DNCertificateClient}. + */ +public class TestDnCertificateClientInit { + + private KeyPair keyPair; + private String certSerialId = "3284792342234"; + private CertificateClient dnCertificateClient; + private HDDSKeyGenerator keyGenerator; + private Path metaDirPath; + private SecurityConfig securityConfig; + private KeyCodec dnKeyCodec; + private X509Certificate x509Certificate; + private static final String DN_COMPONENT = DNCertificateClient.COMPONENT_NAME; + + private static Stream parameters() { + return Stream.of( + arguments(false, false, false, GETCERT), + arguments(false, false, true, FAILURE), + arguments(false, true, false, FAILURE), + arguments(true, false, false, FAILURE), + arguments(false, true, true, FAILURE), + arguments(true, true, false, GETCERT), + arguments(true, false, true, SUCCESS), + arguments(true, true, true, SUCCESS) + ); + } + + @BeforeEach + public void setUp() throws Exception { + OzoneConfiguration config = new OzoneConfiguration(); + final String path = GenericTestUtils + .getTempPath(UUID.randomUUID().toString()); + metaDirPath = Paths.get(path, "test"); + config.set(HDDS_METADATA_DIR_NAME, metaDirPath.toString()); + securityConfig = new SecurityConfig(config); + keyGenerator = new HDDSKeyGenerator(securityConfig); + keyPair = keyGenerator.generateKey(); + x509Certificate = getX509Certificate(); + certSerialId = x509Certificate.getSerialNumber().toString(); + dnCertificateClient = + new DNCertificateClient(securityConfig, null, certSerialId, null, null); + dnKeyCodec = new KeyCodec(securityConfig, DN_COMPONENT); + + Files.createDirectories(securityConfig.getKeyLocation(DN_COMPONENT)); + } + + @AfterEach + public void tearDown() { + dnCertificateClient = null; + FileUtils.deleteQuietly(metaDirPath.toFile()); + } + + + @ParameterizedTest + @MethodSource("parameters") + public void testInitDatanode(boolean pvtKeyPresent, boolean pubKeyPresent, + boolean certPresent, InitResponse expectedResult) throws Exception { + if (pvtKeyPresent) { + dnKeyCodec.writePrivateKey(keyPair.getPrivate()); + } else { + FileUtils.deleteQuietly(Paths.get( + securityConfig.getKeyLocation(DN_COMPONENT).toString(), + securityConfig.getPrivateKeyFileName()).toFile()); + } + + if (pubKeyPresent) { + if (dnCertificateClient.getPublicKey() == null) { + dnKeyCodec.writePublicKey(keyPair.getPublic()); + } + } else { + FileUtils.deleteQuietly( + Paths.get(securityConfig.getKeyLocation(DN_COMPONENT).toString(), + securityConfig.getPublicKeyFileName()).toFile()); + } + + if (certPresent) { + CertificateCodec codec = new CertificateCodec(securityConfig, + DN_COMPONENT); + codec.writeCertificate(new X509CertificateHolder( + x509Certificate.getEncoded())); + } else { + FileUtils.deleteQuietly(Paths.get( + securityConfig.getKeyLocation(DN_COMPONENT).toString(), + securityConfig.getCertificateFileName()).toFile()); + } + InitResponse response = dnCertificateClient.init(); + + assertEquals(expectedResult, response); + + if (!response.equals(FAILURE)) { + assertTrue(OzoneSecurityUtil.checkIfFileExist( + securityConfig.getKeyLocation(DN_COMPONENT), + securityConfig.getPrivateKeyFileName())); + assertTrue(OzoneSecurityUtil.checkIfFileExist( + securityConfig.getKeyLocation(DN_COMPONENT), + securityConfig.getPublicKeyFileName())); + } + } + + private X509Certificate getX509Certificate() throws Exception { + return KeyStoreTestUtil.generateCertificate( + "CN=Test", keyPair, 365, securityConfig.getSignatureAlgo()); + } +} \ No newline at end of file diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index 76776f86a1eb..5952ce5947cc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -24,7 +24,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.security.KeyPair; +import java.security.cert.CertificateExpiredException; import java.security.cert.X509Certificate; +import java.time.Duration; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; @@ -37,6 +39,9 @@ import org.apache.hadoop.hdds.conf.DefaultConfigManager; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; +import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos; +import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto; +import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.ScmConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.ScmInfo; @@ -51,7 +56,10 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.security.exception.SCMSecurityException; import org.apache.hadoop.hdds.security.x509.SecurityConfig; +import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; +import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate; +import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator; import org.apache.hadoop.hdds.security.x509.keys.KeyCodec; import org.apache.hadoop.hdds.utils.HAUtils; @@ -71,11 +79,13 @@ import org.apache.hadoop.ozone.om.helpers.S3SecretValue; import org.apache.hadoop.ozone.om.protocolPB.OmTransportFactory; import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; +import org.apache.hadoop.ozone.security.OMCertificateClient; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.security.KerberosAuthException; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.security.token.Token; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; @@ -85,6 +95,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY; @@ -117,6 +128,7 @@ import org.bouncycastle.asn1.x500.RDN; import org.bouncycastle.asn1.x500.X500Name; import org.bouncycastle.asn1.x500.style.BCStyle; +import org.bouncycastle.cert.X509CertificateHolder; import org.bouncycastle.cert.jcajce.JcaX509CertificateHolder; import org.junit.After; import static org.junit.Assert.assertEquals; @@ -126,13 +138,21 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; + +import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.rules.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import static org.mockito.ArgumentMatchers.anyObject; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.slf4j.event.Level.INFO; /** @@ -195,8 +215,12 @@ public void init() { conf.set(OZONE_METADATA_DIRS, metaDirPath.toString()); conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name()); + conf.set(HDDS_X509_RENEW_GRACE_DURATION, "PT5S"); // 5s workDir = GenericTestUtils.getTestDir(getClass().getSimpleName()); + clusterId = UUID.randomUUID().toString(); + scmId = UUID.randomUUID().toString(); + omId = UUID.randomUUID().toString(); startMiniKdc(); setSecureConfig(); @@ -215,6 +239,9 @@ public void stop() { if (scm != null) { scm.stop(); } + if (om != null) { + om.stop(); + } IOUtils.closeQuietly(om); IOUtils.closeQuietly(omClient); } catch (Exception e) { @@ -372,10 +399,6 @@ public void testAdminAccessControlException() throws Exception { } private void initSCM() throws IOException { - clusterId = UUID.randomUUID().toString(); - scmId = UUID.randomUUID().toString(); - omId = UUID.randomUUID().toString(); - final String path = folder.newFolder().toString(); Path scmPath = Paths.get(path, "scm-meta"); Files.createDirectories(scmPath); @@ -823,6 +846,218 @@ public void testSecureOmInitSuccess() throws Exception { } + /** + * Test successful certificate rotation. + */ + @Test + public void testCertificateRotation() throws Exception { + OMStorage omStorage = new OMStorage(conf); + omStorage.setClusterId(clusterId); + omStorage.setOmId(omId); + OzoneManager.setTestSecureOmFlag(true); + + SecurityConfig securityConfig = new SecurityConfig(conf); + CertificateCodec certCodec = new CertificateCodec(securityConfig, "om"); + OMCertificateClient client = + new OMCertificateClient(securityConfig, omStorage, scmId); + client.init(); + + // save first cert + final int certificateLifetime = 20; // seconds + X509CertificateHolder certHolder = generateX509CertHolder(conf, + new KeyPair(client.getPublicKey(), client.getPrivateKey()), + null, Duration.ofSeconds(certificateLifetime)); + String certId = certHolder.getSerialNumber().toString(); + certCodec.writeCertificate(certHolder); + client.setCertificateId(certId); + omStorage.setOmCertSerialId(certId); + omStorage.forceInitialize(); + + // first renewed cert + X509CertificateHolder newCertHolder = generateX509CertHolder(conf, + null, LocalDateTime.now().plus(securityConfig.getRenewalGracePeriod()), + Duration.ofSeconds(certificateLifetime)); + String pemCert = CertificateCodec.getPEMEncodedString(newCertHolder); + SCMGetCertResponseProto responseProto = SCMGetCertResponseProto.newBuilder() + .setResponseCode(SCMSecurityProtocolProtos + .SCMGetCertResponseProto.ResponseCode.success) + .setX509Certificate(pemCert) + .setX509CACertificate(pemCert) + .build(); + SCMSecurityProtocolClientSideTranslatorPB scmClient = + mock(SCMSecurityProtocolClientSideTranslatorPB.class); + when(scmClient.getOMCertChain(anyObject(), anyString())) + .thenReturn(responseProto); + client.setSecureScmClient(scmClient); + + // create Ozone Manager instance, it will start the monitor task + conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost"); + om = OzoneManager.createOm(conf); + om.setCertClient(client); + + // check after renew, client will have the new cert ID + String id1 = newCertHolder.getSerialNumber().toString(); + GenericTestUtils.waitFor(() -> + id1.equals(client.getCertificate().getSerialNumber().toString()), + 1000, certificateLifetime * 1000); + + // test the second time certificate rotation + // second renewed cert + newCertHolder = generateX509CertHolder(conf, + null, null, Duration.ofSeconds(certificateLifetime)); + pemCert = CertificateCodec.getPEMEncodedString(newCertHolder); + responseProto = SCMGetCertResponseProto.newBuilder() + .setResponseCode(SCMSecurityProtocolProtos + .SCMGetCertResponseProto.ResponseCode.success) + .setX509Certificate(pemCert) + .setX509CACertificate(pemCert) + .build(); + when(scmClient.getOMCertChain(anyObject(), anyString())) + .thenReturn(responseProto); + String id2 = newCertHolder.getSerialNumber().toString(); + + // check after renew, client will have the new cert ID + GenericTestUtils.waitFor(() -> + id2.equals(client.getCertificate().getSerialNumber().toString()), + 1000, certificateLifetime * 1000); + } + /** + * Test unexpected SCMGetCertResponseProto returned from SCM. + */ + @Test + public void testCertificateRotationRecoverableFailure() throws Exception { + LogCapturer omLogs = LogCapturer.captureLogs(OMCertificateClient.LOG); + OMStorage omStorage = new OMStorage(conf); + omStorage.setClusterId(clusterId); + omStorage.setOmId(omId); + OzoneManager.setTestSecureOmFlag(true); + + SecurityConfig securityConfig = new SecurityConfig(conf); + CertificateCodec certCodec = new CertificateCodec(securityConfig, "om"); + OMCertificateClient client = + new OMCertificateClient(securityConfig, omStorage, scmId); + client.init(); + + // save first cert + final int certificateLifetime = 20; // seconds + X509CertificateHolder certHolder = generateX509CertHolder(conf, + new KeyPair(client.getPublicKey(), client.getPrivateKey()), + null, Duration.ofSeconds(certificateLifetime)); + String certId = certHolder.getSerialNumber().toString(); + certCodec.writeCertificate(certHolder); + client.setCertificateId(certId); + omStorage.setOmCertSerialId(certId); + omStorage.forceInitialize(); + + // prepare a mocked scmClient to certificate signing + SCMSecurityProtocolClientSideTranslatorPB scmClient = + mock(SCMSecurityProtocolClientSideTranslatorPB.class); + client.setSecureScmClient(scmClient); + + Duration gracePeriod = securityConfig.getRenewalGracePeriod(); + X509CertificateHolder newCertHolder = generateX509CertHolder(conf, null, + LocalDateTime.now().plus(gracePeriod), + Duration.ofSeconds(certificateLifetime)); + String pemCert = CertificateCodec.getPEMEncodedString(newCertHolder); + // provide an invalid SCMGetCertResponseProto. Without + // setX509CACertificate(pemCert), signAndStoreCert will throw exception. + SCMSecurityProtocolProtos.SCMGetCertResponseProto responseProto = + SCMSecurityProtocolProtos.SCMGetCertResponseProto + .newBuilder().setResponseCode(SCMSecurityProtocolProtos + .SCMGetCertResponseProto.ResponseCode.success) + .setX509Certificate(pemCert) + .build(); + when(scmClient.getOMCertChain(anyObject(), anyString())) + .thenReturn(responseProto); + + // check that new cert ID should not equal to current cert ID + String certId1 = newCertHolder.getSerialNumber().toString(); + Assert.assertFalse(certId1.equals( + client.getCertificate().getSerialNumber().toString())); + + // certificate failed to renew, client still hold the old expired cert. + Thread.sleep(certificateLifetime * 1000); + Assert.assertTrue(certId.equals( + client.getCertificate().getSerialNumber().toString())); + try { + client.getCertificate().checkValidity(); + } catch (Exception e) { + Assert.assertTrue(e instanceof CertificateExpiredException); + } + Assert.assertTrue(omLogs.getOutput().contains( + "Error while signing and storing SCM signed certificate.")); + + // provide a new valid SCMGetCertResponseProto + newCertHolder = generateX509CertHolder(conf, null, null, + Duration.ofSeconds(certificateLifetime)); + pemCert = CertificateCodec.getPEMEncodedString(newCertHolder); + responseProto = SCMSecurityProtocolProtos.SCMGetCertResponseProto + .newBuilder().setResponseCode(SCMSecurityProtocolProtos + .SCMGetCertResponseProto.ResponseCode.success) + .setX509Certificate(pemCert) + .setX509CACertificate(pemCert) + .build(); + when(scmClient.getOMCertChain(anyObject(), anyString())) + .thenReturn(responseProto); + String certId2 = newCertHolder.getSerialNumber().toString(); + + // check after renew, client will have the new cert ID + GenericTestUtils.waitFor(() -> { + String newCertId = client.getCertificate().getSerialNumber().toString(); + return newCertId.equals(certId2); + }, 1000, certificateLifetime * 1000); + } + + /** + * Test the directory rollback failure case. + */ + @Test + @Ignore("Run it locally since it will terminate the process.") + public void testCertificateRotationUnRecoverableFailure() throws Exception { + LogCapturer omLogs = LogCapturer.captureLogs(OzoneManager.getLogger()); + OMStorage omStorage = new OMStorage(conf); + omStorage.setClusterId(clusterId); + omStorage.setOmId(omId); + OzoneManager.setTestSecureOmFlag(true); + + SecurityConfig securityConfig = new SecurityConfig(conf); + CertificateCodec certCodec = new CertificateCodec(securityConfig, "om"); + OMCertificateClient client = + new OMCertificateClient(securityConfig, omStorage, scmId); + client.init(); + + // save first cert + final int certificateLifetime = 20; // seconds + X509CertificateHolder certHolder = generateX509CertHolder(conf, + new KeyPair(client.getPublicKey(), client.getPrivateKey()), + null, Duration.ofSeconds(certificateLifetime)); + String certId = certHolder.getSerialNumber().toString(); + certCodec.writeCertificate(certHolder); + omStorage.setOmCertSerialId(certId); + omStorage.forceInitialize(); + + X509CertificateHolder newCertHolder = generateX509CertHolder(conf, null, + null, Duration.ofSeconds(certificateLifetime)); + DNCertificateClient mockClient = mock(DNCertificateClient.class); + when(mockClient.getCertificate()).thenReturn( + CertificateCodec.getX509Certificate(newCertHolder)); + when(mockClient.timeBeforeExpiryGracePeriod(anyObject())) + .thenReturn(Duration.ZERO); + when(mockClient.renewAndStoreKeyAndCertificate(anyObject())).thenThrow( + new CertificateException("renewAndStoreKeyAndCert failed ", + CertificateException.ErrorCode.ROLLBACK_ERROR)); + + // create Ozone Manager instance, it will start the monitor task + conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost"); + om = OzoneManager.createOm(conf); + om.setCertClient(mockClient); + + // check error message during renew + GenericTestUtils.waitFor(() -> omLogs.getOutput().contains( + "OzoneManage shutdown because certificate rollback failure."), + 1000, certificateLifetime * 1000); + } + public void validateCertificate(X509Certificate cert) throws Exception { // Assert that we indeed have a self signed certificate. @@ -871,4 +1106,23 @@ private void initializeOmStorage(OMStorage omStorage) throws IOException { } omStorage.initialize(); } + + private static X509CertificateHolder generateX509CertHolder( + OzoneConfiguration conf, KeyPair keyPair, LocalDateTime startDate, + Duration certLifetime) throws Exception { + if (keyPair == null) { + keyPair = KeyStoreTestUtil.generateKeyPair("RSA"); + } + LocalDateTime start = startDate == null ? LocalDateTime.now() : startDate; + LocalDateTime end = start.plus(certLifetime); + return SelfSignedCertificate.newBuilder() + .setBeginDate(start) + .setEndDate(end) + .setClusterID("cluster") + .setKey(keyPair) + .setSubject("localhost") + .setConfiguration(conf) + .setScmID("test") + .build(); + } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java index 87221827e258..aed1a3852233 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.file.Path; import java.security.KeyPair; import java.security.PrivateKey; import java.security.PublicKey; @@ -45,6 +46,7 @@ import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil; import org.bouncycastle.cert.X509CertificateHolder; import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION_DEFAULT; @@ -68,7 +70,6 @@ public class CertificateClientTestImpl implements CertificateClient { private DefaultApprover approver; private KeyStoresFactory serverKeyStoresFactory; private KeyStoresFactory clientKeyStoresFactory; - private boolean isKeyRenewed = false; public CertificateClientTestImpl(OzoneConfiguration conf) throws Exception { this(conf, true); @@ -169,6 +170,10 @@ public boolean verifyCertificate(X509Certificate certificate) { return true; } + @Override + public void setCertificateId(String certSerialId) { + } + @Override public byte[] signDataStream(InputStream stream) throws CertificateException { @@ -192,11 +197,29 @@ public boolean verifySignature(byte[] data, byte[] signature, return true; } + @Override + public CertificateSignRequest.Builder getCSRBuilder(KeyPair key) + throws CertificateException { + return null; + } + @Override public CertificateSignRequest.Builder getCSRBuilder() { return new CertificateSignRequest.Builder(); } + @Override + public String signAndStoreCertificate(PKCS10CertificationRequest request, + Path certPath) throws CertificateException { + return null; + } + + @Override + public String signAndStoreCertificate(PKCS10CertificationRequest request) + throws CertificateException { + return null; + } + @Override public X509Certificate queryCertificate(String query) { return null; @@ -297,10 +320,6 @@ public boolean processCrl(CRLInfo crl) { return false; } - public boolean isCertificateRenewed() { - return isKeyRenewed; - } - public void renewKey() throws Exception { KeyPair newKeyPair = keyGen.generateKey(); CertificateSignRequest.Builder csrBuilder = getCSRBuilder(); @@ -327,7 +346,6 @@ public void renewKey() throws Exception { // Save certificate and private key to keyStore keyPair = newKeyPair; x509Certificate = newX509Certificate; - isKeyRenewed = true; System.out.println(new Date() + " certificated is renewed"); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java index 6970dea06ca4..ebb8e97ba44c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java @@ -58,6 +58,7 @@ import java.io.File; import java.nio.file.Path; import java.security.cert.CertificateExpiredException; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -97,6 +98,7 @@ public class TestOzoneContainerWithTLS { private ContainerTokenSecretManager secretManager; private CertificateClientTestImpl caClient; private boolean containerTokenEnabled; + private int certLifetime = 10 * 1000; // 10s public TestOzoneContainerWithTLS(boolean enableToken) { this.containerTokenEnabled = enableToken; @@ -129,8 +131,9 @@ public void setup() throws Exception { conf.setBoolean(HddsConfigKeys.HDDS_GRPC_TLS_TEST_CERT, true); conf.setInt(HDDS_KEY_LEN, 1024); - // certificate lives for 5s - conf.set(HDDS_X509_DEFAULT_DURATION, "PT5S"); + // certificate lives for 10s + conf.set(HDDS_X509_DEFAULT_DURATION, + Duration.ofMillis(certLifetime).toString()); conf.set(HDDS_SECURITY_SSL_KEYSTORE_RELOAD_INTERVAL, "1s"); conf.set(HDDS_SECURITY_SSL_TRUSTSTORE_RELOAD_INTERVAL, "1s"); @@ -146,7 +149,7 @@ public void setup() throws Exception { @Test(expected = CertificateExpiredException.class) public void testCertificateLifetime() throws Exception { // Sleep to wait for certificate expire - Thread.sleep(5000); + Thread.sleep(certLifetime); caClient.getCertificate().checkValidity(); } @@ -244,7 +247,7 @@ public void testContainerDownload() throws Exception { // Wait certificate to expire GenericTestUtils.waitFor(() -> caClient.getCertificate().getNotAfter().before(new Date()), - 500, 5000); + 500, certLifetime); List sourceDatanodes = new ArrayList<>(); sourceDatanodes.add(dn); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java index 1cafedc32b75..18ca87994052 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java @@ -87,7 +87,8 @@ public class TestContainerServer { public static void setup() { DefaultMetricsSystem.setMiniClusterMode(true); CONF.set(HddsConfigKeys.HDDS_METADATA_DIR_NAME, TEST_DIR); - caClient = new DNCertificateClient(new SecurityConfig(CONF)); + caClient = new DNCertificateClient(new SecurityConfig(CONF), + null, null, null, null); } @Test diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestSecureOzoneManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestSecureOzoneManager.java index 6347a4f9657e..1871fb1bc217 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestSecureOzoneManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestSecureOzoneManager.java @@ -22,10 +22,10 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; -import org.apache.hadoop.hdds.security.x509.certificate.client.OMCertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.security.x509.keys.KeyCodec; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.security.OMCertificateClient; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.ozone.test.LambdaTestUtils; import org.bouncycastle.cert.X509CertificateHolder; @@ -121,8 +121,8 @@ public void testSecureOmInitFailures() throws Exception { // Case 1: When keypair as well as certificate is missing. Initial keypair // boot-up. Get certificate will fail when SCM is not running. SecurityConfig securityConfig = new SecurityConfig(conf); - CertificateClient client = new OMCertificateClient(securityConfig, - omStorage.getOmCertSerialId()); + CertificateClient client = + new OMCertificateClient(securityConfig, omStorage, scmId); Assert.assertEquals(CertificateClient.InitResponse.GETCERT, client.init()); privateKey = client.getPrivateKey(); publicKey = client.getPublicKey(); @@ -131,8 +131,7 @@ public void testSecureOmInitFailures() throws Exception { Assert.assertNull(client.getCertificate()); // Case 2: If key pair already exist than response should be RECOVER. - client = new OMCertificateClient(securityConfig, - omStorage.getOmCertSerialId()); + client = new OMCertificateClient(securityConfig, omStorage, scmId); Assert.assertEquals(CertificateClient.InitResponse.RECOVER, client.init()); Assert.assertNotNull(client.getPrivateKey()); Assert.assertNotNull(client.getPublicKey()); @@ -168,17 +167,15 @@ public void testSecureOmInitFailures() throws Exception { securityConfig.getSignatureAlgo()); certCodec.writeCertificate(new X509CertificateHolder( x509Certificate.getEncoded())); - client = new OMCertificateClient(securityConfig, - x509Certificate.getSerialNumber().toString()); omStorage.setOmCertSerialId(x509Certificate.getSerialNumber().toString()); + client = new OMCertificateClient(securityConfig, omStorage, scmId); Assert.assertEquals(CertificateClient.InitResponse.FAILURE, client.init()); Assert.assertNull(client.getPrivateKey()); Assert.assertNull(client.getPublicKey()); Assert.assertNotNull(client.getCertificate()); // Case 6: When private key and certificate is present. - client = new OMCertificateClient(securityConfig, - x509Certificate.getSerialNumber().toString()); + client = new OMCertificateClient(securityConfig, omStorage, scmId); FileUtils.deleteQuietly(Paths.get(securityConfig.getKeyLocation(COMPONENT) .toString(), securityConfig.getPublicKeyFileName()).toFile()); keyCodec.writePrivateKey(privateKey); @@ -188,8 +185,7 @@ public void testSecureOmInitFailures() throws Exception { Assert.assertNotNull(client.getCertificate()); // Case 7 When keypair and certificate is present. - client = new OMCertificateClient(securityConfig, - x509Certificate.getSerialNumber().toString()); + client = new OMCertificateClient(securityConfig, omStorage, scmId); Assert.assertEquals(CertificateClient.InitResponse.SUCCESS, client.init()); Assert.assertNotNull(client.getPrivateKey()); Assert.assertNotNull(client.getPublicKey()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 71deb38cb6e3..8d2553f1edb3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -30,9 +30,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.security.KeyPair; import java.security.PrivilegedExceptionAction; -import java.security.cert.CertificateException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -65,8 +63,6 @@ import org.apache.hadoop.hdds.conf.ConfigurationException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto; -import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.server.OzoneAdmins; @@ -83,7 +79,7 @@ import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; -import org.apache.hadoop.hdds.security.x509.certificate.client.OMCertificateClient; +import org.apache.hadoop.ozone.security.OMCertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl; @@ -215,7 +211,6 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT; import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients; -import static org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString; import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName; import static org.apache.hadoop.hdds.server.ServerUtils.updateRPCListenAddress; import static org.apache.hadoop.hdds.utils.HAUtils.getScmInfo; @@ -274,6 +269,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PERMISSION_DENIED; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_ERROR_OTHER; +import static org.apache.hadoop.util.ExitUtil.terminate; import static org.apache.hadoop.util.MetricUtil.captureLatencyNs; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.VOLUME_LOCK; @@ -291,7 +287,6 @@ import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.FileUtils; import org.apache.ratis.util.LifeCycle; -import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -544,13 +539,16 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) // For testing purpose only, not hit scm from om as Hadoop UGI can't login // two principals in the same JVM. + ScmInfo scmInfo; if (!testSecureOmFlag) { - ScmInfo scmInfo = getScmInfo(configuration); + scmInfo = getScmInfo(configuration); if (!scmInfo.getClusterId().equals(omStorage.getClusterID())) { logVersionMismatch(conf, scmInfo); throw new OMException("SCM version info mismatch.", ResultCodes.SCM_VERSION_MISMATCH_ERROR); } + } else { + scmInfo = new ScmInfo.Builder().setScmId("testSecureOm").build(); } RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class, @@ -570,8 +568,9 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) throw new RuntimeException("OzoneManager started in secure mode but " + "doesn't have SCM signed certificate."); } - certClient = new OMCertificateClient(new SecurityConfig(conf), - omStorage.getOmCertSerialId()); + certClient = new OMCertificateClient(secConfig, omStorage, + scmInfo == null ? null : scmInfo.getScmId(), this::saveNewCertId, + this::terminateOM); } if (secConfig.isBlockTokenEnabled()) { blockTokenMgr = createBlockTokenSecretManager(configuration); @@ -1037,6 +1036,7 @@ public void startSecretManager() { /** * For testing purpose only. */ + @VisibleForTesting public void setCertClient(CertificateClient certClient) { // TODO: Initialize it in constructor with implementation for certClient. this.certClient = certClient; @@ -1278,19 +1278,18 @@ public static boolean omInit(OzoneConfiguration conf) throws IOException, */ @VisibleForTesting public static void initializeSecurity(OzoneConfiguration conf, - OMStorage omStore, String scmId) - throws IOException { + OMStorage omStore, String scmId) throws IOException { LOG.info("Initializing secure OzoneManager."); CertificateClient certClient = - new OMCertificateClient(new SecurityConfig(conf), - omStore.getOmCertSerialId()); + new OMCertificateClient(new SecurityConfig(conf), omStore, scmId); CertificateClient.InitResponse response = certClient.init(); if (response.equals(CertificateClient.InitResponse.REINIT)) { LOG.info("Re-initialize certificate client."); omStore.unsetOmCertSerialId(); omStore.persistCurrentState(); - certClient = new OMCertificateClient(new SecurityConfig(conf)); + certClient = new OMCertificateClient( + new SecurityConfig(conf), omStore, scmId); response = certClient.init(); } LOG.info("Init response: {}", response); @@ -1299,7 +1298,10 @@ public static void initializeSecurity(OzoneConfiguration conf, LOG.info("Initialization successful."); break; case GETCERT: - getSCMSignedCert(certClient, conf, omStore, scmId); + // Sign and persist OM cert. + CertificateSignRequest.Builder builder = certClient.getCSRBuilder(); + omStore.setOmCertSerialId( + certClient.signAndStoreCertificate(builder.build())); LOG.info("Successfully stored SCM signed certificate."); break; case FAILURE: @@ -2096,6 +2098,11 @@ public void shutDown(String message) { ExitUtils.terminate(0, message, LOG); } + public void terminateOM() { + stop(); + terminate(1); + } + /** * Wait until service has completed shutdown. */ @@ -2119,120 +2126,6 @@ private void startSecretManagerIfNecessary() { } } - /** - * Get SCM signed certificate and store it using certificate client. - */ - private static void getSCMSignedCert(CertificateClient client, - OzoneConfiguration config, OMStorage omStore, String scmId) - throws IOException { - CertificateSignRequest.Builder builder = client.getCSRBuilder(); - KeyPair keyPair = new KeyPair(client.getPublicKey(), - client.getPrivateKey()); - boolean flexibleFqdnResolutionEnabled = config.getBoolean( - OZONE_FLEXIBLE_FQDN_RESOLUTION_ENABLED, - OZONE_FLEXIBLE_FQDN_RESOLUTION_ENABLED_DEFAULT); - InetSocketAddress omRpcAdd = OmUtils.getOmAddress(config); - String ip = null; - - boolean addressResolved = omRpcAdd != null && omRpcAdd.getAddress() != null; - if (flexibleFqdnResolutionEnabled && !addressResolved && omRpcAdd != null) { - InetSocketAddress omRpcAddWithHostName = - OzoneNetUtils.getAddressWithHostNameLocal(omRpcAdd); - if (omRpcAddWithHostName != null - && omRpcAddWithHostName.getAddress() != null) { - addressResolved = true; - ip = omRpcAddWithHostName.getAddress().getHostAddress(); - } - } - - if (!addressResolved) { - LOG.error("Incorrect om rpc address. omRpcAdd:{}", omRpcAdd); - throw new RuntimeException("Can't get SCM signed certificate. " + - "omRpcAdd: " + omRpcAdd); - } - - if (ip == null) { - ip = omRpcAdd.getAddress().getHostAddress(); - } - - String hostname = omRpcAdd.getHostName(); - int port = omRpcAdd.getPort(); - String subject; - if (builder.hasDnsName()) { - subject = UserGroupInformation.getCurrentUser().getShortUserName() - + "@" + hostname; - } else { - // With only IP in alt.name, certificate validation would fail if subject - // isn't a hostname either, so omit username. - subject = hostname; - } - - builder.setCA(false) - .setKey(keyPair) - .setConfiguration(config) - .setScmID(scmId) - .setClusterID(omStore.getClusterID()) - .setSubject(subject); - - OMHANodeDetails haOMHANodeDetails = OMHANodeDetails.loadOMHAConfig(config); - String serviceName = - haOMHANodeDetails.getLocalNodeDetails().getServiceId(); - if (!StringUtils.isEmpty(serviceName)) { - builder.addServiceName(serviceName); - } - - LOG.info("Creating csr for OM->dns:{},ip:{},scmId:{},clusterId:{}," + - "subject:{}", hostname, ip, scmId, omStore.getClusterID(), subject); - - HddsProtos.OzoneManagerDetailsProto.Builder omDetailsProtoBuilder = - HddsProtos.OzoneManagerDetailsProto.newBuilder() - .setHostName(hostname) - .setIpAddress(ip) - .setUuid(omStore.getOmId()) - .addPorts(HddsProtos.Port.newBuilder() - .setName(RPC_PORT) - .setValue(port) - .build()); - - PKCS10CertificationRequest csr = builder.build(); - HddsProtos.OzoneManagerDetailsProto omDetailsProto = - omDetailsProtoBuilder.build(); - LOG.info("OzoneManager ports added:{}", omDetailsProto.getPortsList()); - SCMSecurityProtocolClientSideTranslatorPB secureScmClient = - HddsServerUtil.getScmSecurityClientWithFixedDuration(config); - - SCMGetCertResponseProto response = secureScmClient. - getOMCertChain(omDetailsProto, getEncodedString(csr)); - String pemEncodedCert = response.getX509Certificate(); - - try { - - // Store SCM CA certificate. - if (response.hasX509CACertificate()) { - String pemEncodedRootCert = response.getX509CACertificate(); - client.storeCertificate(pemEncodedRootCert, true, true); - client.storeCertificate(pemEncodedCert, true); - - // Store Root CA certificate if available. - if (response.hasX509RootCACertificate()) { - client.storeRootCACertificate(response.getX509RootCACertificate(), - true); - } - - // Persist om cert serial id. - omStore.setOmCertSerialId(CertificateCodec. - getX509Certificate(pemEncodedCert).getSerialNumber().toString()); - } else { - throw new RuntimeException("Unable to retrieve OM certificate " + - "chain"); - } - } catch (IOException | CertificateException e) { - LOG.error("Error while storing SCM signed certificate.", e); - throw new RuntimeException(e); - } - - } - /** * @return true if delegation token operation is allowed */ @@ -4572,4 +4465,64 @@ private void updateLayoutVersionInDB(OMLayoutVersionManager lvm, private BucketLayout getBucketLayout() { return BucketLayout.DEFAULT; } + + void saveNewCertId(String certId) { + try { + omStorage.setOmCertSerialId(certId); + omStorage.persistCurrentState(); + } catch (IOException ex) { + // New cert ID cannot be persisted into VERSION file. + LOG.error("Failed to persist new cert ID {} to VERSION file." + + "Terminating OzoneManager...", certId, ex); + shutDown("OzoneManage shutdown because VERSION file persist failure."); + } + } + + public static HddsProtos.OzoneManagerDetailsProto getOmDetailsProto( + OzoneConfiguration config, String omID) { + boolean flexibleFqdnResolutionEnabled = config.getBoolean( + OZONE_FLEXIBLE_FQDN_RESOLUTION_ENABLED, + OZONE_FLEXIBLE_FQDN_RESOLUTION_ENABLED_DEFAULT); + InetSocketAddress omRpcAdd = OmUtils.getOmAddress(config); + String ip = null; + + boolean addressResolved = omRpcAdd != null && omRpcAdd.getAddress() != null; + if (flexibleFqdnResolutionEnabled && !addressResolved && omRpcAdd != null) { + InetSocketAddress omRpcAddWithHostName = + OzoneNetUtils.getAddressWithHostNameLocal(omRpcAdd); + if (omRpcAddWithHostName != null + && omRpcAddWithHostName.getAddress() != null) { + addressResolved = true; + ip = omRpcAddWithHostName.getAddress().getHostAddress(); + } + } + + if (!addressResolved) { + LOG.error("Incorrect om rpc address. omRpcAdd:{}", omRpcAdd); + throw new RuntimeException("Can't get SCM signed certificate. " + + "omRpcAdd: " + omRpcAdd); + } + + if (ip == null) { + ip = omRpcAdd.getAddress().getHostAddress(); + } + + String hostname = omRpcAdd.getHostName(); + int port = omRpcAdd.getPort(); + + HddsProtos.OzoneManagerDetailsProto.Builder omDetailsProtoBuilder = + HddsProtos.OzoneManagerDetailsProto.newBuilder() + .setHostName(hostname) + .setIpAddress(ip) + .setUuid(omID) + .addPorts(HddsProtos.Port.newBuilder() + .setName(RPC_PORT) + .setValue(port) + .build()); + + HddsProtos.OzoneManagerDetailsProto omDetailsProto = + omDetailsProtoBuilder.build(); + LOG.info("OzoneManager ports added:{}", omDetailsProto.getPortsList()); + return omDetailsProto; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OMCertificateClient.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OMCertificateClient.java new file mode 100644 index 000000000000..ceb449b8fb18 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OMCertificateClient.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.security; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto; +import org.apache.hadoop.hdds.security.x509.SecurityConfig; +import org.apache.hadoop.hdds.security.x509.certificate.client.CommonCertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; +import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; +import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; +import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ha.OMHANodeDetails; +import org.apache.hadoop.security.UserGroupInformation; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.security.KeyPair; +import java.util.function.Consumer; + +import static org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString; + +/** + * Certificate client for OzoneManager. + */ +public class OMCertificateClient extends CommonCertificateClient { + + public static final Logger LOG = + LoggerFactory.getLogger(OMCertificateClient.class); + + public static final String COMPONENT_NAME = "om"; + private String scmID; + private final String clusterID; + private final HddsProtos.OzoneManagerDetailsProto omInfo; + + @SuppressWarnings("parameternumber") + public OMCertificateClient(SecurityConfig secConfig, String scmId, + String clusterId, HddsProtos.OzoneManagerDetailsProto omDetails, + String certSerialId, String localCrlId, + Consumer persistCertIdCallback, Runnable shutdownCallback) { + super(secConfig, LOG, certSerialId, COMPONENT_NAME, persistCertIdCallback, + shutdownCallback); + this.setLocalCrlId(localCrlId != null ? + Long.parseLong(localCrlId) : 0); + this.scmID = scmId; + this.clusterID = clusterId; + this.omInfo = omDetails; + } + + public OMCertificateClient(SecurityConfig secConfig, + OMStorage omStorage, String scmID, Consumer saveCertIdCallback, + Runnable shutdownCallback) { + this(secConfig, scmID, omStorage.getClusterID(), + OzoneManager.getOmDetailsProto( + (OzoneConfiguration) secConfig.getConfiguration(), + omStorage.getOmId()), + omStorage.getOmCertSerialId(), null, + saveCertIdCallback, shutdownCallback); + } + + public OMCertificateClient(SecurityConfig secConfig, OMStorage omStorage, + String scmID) { + this(secConfig, scmID, omStorage.getClusterID(), + OzoneManager.getOmDetailsProto( + (OzoneConfiguration) secConfig.getConfiguration(), + omStorage.getOmId()), + omStorage.getOmCertSerialId(), null, null, null); + } + + public OMCertificateClient(SecurityConfig secConfig) { + this(secConfig, null, null, null, null, null, null, null); + } + + public OMCertificateClient(SecurityConfig secConfig, String certSerialId) { + this(secConfig, null, null, null, certSerialId, null, null, null); + } + + /** + * Returns a CSR builder that can be used to create a Certificate signing + * request. + * The default flag is added to allow basic SSL handshake. + * + * @return CertificateSignRequest.Builder + */ + @Override + public CertificateSignRequest.Builder getCSRBuilder() + throws CertificateException { + return getCSRBuilder(new KeyPair(getPublicKey(), getPrivateKey())); + } + + /** + * Returns a CSR builder that can be used to create a Certificate sigining + * request. + * + * @return CertificateSignRequest.Builder + */ + @Override + public CertificateSignRequest.Builder getCSRBuilder(KeyPair keyPair) + throws CertificateException { + CertificateSignRequest.Builder builder = super.getCSRBuilder() + .setDigitalEncryption(true) + .setDigitalSignature(true); + + String hostname = omInfo.getHostName(); + String subject; + if (builder.hasDnsName()) { + try { + subject = UserGroupInformation.getCurrentUser().getShortUserName() + + "@" + hostname; + } catch (IOException e) { + throw new CertificateException("Failed to getCurrentUser", e); + } + } else { + // With only IP in alt.name, certificate validation would fail if subject + // isn't a hostname either, so omit username. + subject = hostname; + } + + builder.setCA(false) + .setKey(keyPair) + .setConfiguration(getConfig()) + .setScmID(scmID) + .setClusterID(clusterID) + .setSubject(subject); + + OMHANodeDetails haOMHANodeDetails = + OMHANodeDetails.loadOMHAConfig(getConfig()); + String serviceName = + haOMHANodeDetails.getLocalNodeDetails().getServiceId(); + if (!StringUtils.isEmpty(serviceName)) { + builder.addServiceName(serviceName); + } + + LOG.info("Creating csr for OM->dns:{},ip:{},scmId:{},clusterId:{}," + + "subject:{}", hostname, omInfo.getIpAddress(), scmID, clusterID, + subject); + return builder; + } + + @Override + public String signAndStoreCertificate(PKCS10CertificationRequest request, + Path certPath) throws CertificateException { + try { + SCMGetCertResponseProto response = getScmSecureClient() + .getOMCertChain(omInfo, getEncodedString(request)); + + String pemEncodedCert = response.getX509Certificate(); + CertificateCodec certCodec = new CertificateCodec( + getSecurityConfig(), certPath); + + // Store SCM CA certificate. + if (response.hasX509CACertificate()) { + String pemEncodedRootCert = response.getX509CACertificate(); + storeCertificate(pemEncodedRootCert, + true, true, false, certCodec, false); + storeCertificate(pemEncodedCert, true, false, false, certCodec, false); + + // Store Root CA certificate if available. + if (response.hasX509RootCACertificate()) { + storeCertificate(response.getX509RootCACertificate(), + true, false, true, certCodec, false); + } + return CertificateCodec.getX509Certificate(pemEncodedCert) + .getSerialNumber().toString(); + } else { + throw new CertificateException("Unable to retrieve OM certificate " + + "chain."); + } + } catch (IOException | java.security.cert.CertificateException e) { + LOG.error("Error while signing and storing SCM signed certificate.", e); + throw new CertificateException( + "Error while signing and storing SCM signed certificate.", e); + } + } + + @Override + public Logger getLogger() { + return LOG; + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerRatisServer.java index 365eb60bfe36..94eb040795bc 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerRatisServer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerRatisServer.java @@ -30,7 +30,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.security.x509.SecurityConfig; -import org.apache.hadoop.hdds.security.x509.certificate.client.OMCertificateClient; +import org.apache.hadoop.ozone.security.OMCertificateClient; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.ha.ratis.RatisSnapshotInfo; diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestCertificateClientInit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOmCertificateClientInit.java similarity index 73% rename from hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestCertificateClientInit.java rename to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOmCertificateClientInit.java index 72730858fc6f..d37af74e88a3 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/client/TestCertificateClientInit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOmCertificateClientInit.java @@ -16,11 +16,12 @@ * limitations under the License. * */ -package org.apache.hadoop.hdds.security.x509.certificate.client; +package org.apache.hadoop.ozone.security; import org.apache.commons.io.FileUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.x509.SecurityConfig; +import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator; import org.apache.hadoop.hdds.security.x509.keys.KeyCodec; @@ -53,21 +54,18 @@ import static org.junit.jupiter.params.provider.Arguments.arguments; /** - * Test class for {@link DefaultCertificateClient}. + * Test class for {@link OMCertificateClient}. */ -public class TestCertificateClientInit { +public class TestOmCertificateClientInit { private KeyPair keyPair; private String certSerialId = "3284792342234"; - private CertificateClient dnCertificateClient; private CertificateClient omCertificateClient; private HDDSKeyGenerator keyGenerator; private Path metaDirPath; private SecurityConfig securityConfig; - private KeyCodec dnKeyCodec; private KeyCodec omKeyCodec; private X509Certificate x509Certificate; - private static final String DN_COMPONENT = DNCertificateClient.COMPONENT_NAME; private static final String OM_COMPONENT = OMCertificateClient.COMPONENT_NAME; private static Stream parameters() { @@ -95,71 +93,18 @@ public void setUp() throws Exception { keyPair = keyGenerator.generateKey(); x509Certificate = getX509Certificate(); certSerialId = x509Certificate.getSerialNumber().toString(); - dnCertificateClient = new DNCertificateClient(securityConfig, - certSerialId); - omCertificateClient = new OMCertificateClient(securityConfig, - certSerialId); - dnKeyCodec = new KeyCodec(securityConfig, DN_COMPONENT); + omCertificateClient = new OMCertificateClient(securityConfig, certSerialId); omKeyCodec = new KeyCodec(securityConfig, OM_COMPONENT); - Files.createDirectories(securityConfig.getKeyLocation(DN_COMPONENT)); Files.createDirectories(securityConfig.getKeyLocation(OM_COMPONENT)); } @AfterEach public void tearDown() { - dnCertificateClient = null; omCertificateClient = null; FileUtils.deleteQuietly(metaDirPath.toFile()); } - - @ParameterizedTest - @MethodSource("parameters") - public void testInitDatanode(boolean pvtKeyPresent, boolean pubKeyPresent, - boolean certPresent, InitResponse expectedResult) throws Exception { - if (pvtKeyPresent) { - dnKeyCodec.writePrivateKey(keyPair.getPrivate()); - } else { - FileUtils.deleteQuietly(Paths.get( - securityConfig.getKeyLocation(DN_COMPONENT).toString(), - securityConfig.getPrivateKeyFileName()).toFile()); - } - - if (pubKeyPresent) { - if (dnCertificateClient.getPublicKey() == null) { - dnKeyCodec.writePublicKey(keyPair.getPublic()); - } - } else { - FileUtils.deleteQuietly( - Paths.get(securityConfig.getKeyLocation(DN_COMPONENT).toString(), - securityConfig.getPublicKeyFileName()).toFile()); - } - - if (certPresent) { - CertificateCodec codec = new CertificateCodec(securityConfig, - DN_COMPONENT); - codec.writeCertificate(new X509CertificateHolder( - x509Certificate.getEncoded())); - } else { - FileUtils.deleteQuietly(Paths.get( - securityConfig.getKeyLocation(DN_COMPONENT).toString(), - securityConfig.getCertificateFileName()).toFile()); - } - InitResponse response = dnCertificateClient.init(); - - assertEquals(expectedResult, response); - - if (!response.equals(FAILURE)) { - assertTrue(OzoneSecurityUtil.checkIfFileExist( - securityConfig.getKeyLocation(DN_COMPONENT), - securityConfig.getPrivateKeyFileName())); - assertTrue(OzoneSecurityUtil.checkIfFileExist( - securityConfig.getKeyLocation(DN_COMPONENT), - securityConfig.getPublicKeyFileName())); - } - } - @ParameterizedTest @MethodSource("parameters") public void testInitOzoneManager(boolean pvtKeyPresent, boolean pubKeyPresent, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java index 4814e8f783b3..0003c73efc3c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/TestOzoneDelegationTokenSecretManager.java @@ -31,7 +31,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; -import org.apache.hadoop.hdds.security.x509.certificate.client.OMCertificateClient; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.OzoneConsts; diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index 154e6a1db49f..a7dacae03352 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -25,9 +25,6 @@ import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos; -import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.recon.ReconConfig; import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.security.x509.SecurityConfig; @@ -48,22 +45,18 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.hadoop.ozone.recon.codegen.ReconSchemaGenerationModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.net.InetAddress; import java.net.InetSocketAddress; -import java.security.cert.CertificateException; import static org.apache.hadoop.hdds.recon.ReconConfig.ConfigStrings.OZONE_RECON_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.hdds.recon.ReconConfig.ConfigStrings.OZONE_RECON_KERBEROS_PRINCIPAL_KEY; -import static org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec.getX509Certificate; -import static org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString; import static org.apache.hadoop.ozone.common.Storage.StorageState.INITIALIZED; import static org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY; +import static org.apache.hadoop.util.ExitUtil.terminate; /** * Recon server main class that stops and starts recon services. @@ -102,8 +95,7 @@ public Void call() throws Exception { configuration = createOzoneConfiguration(); ConfigurationProvider.setConfiguration(configuration); - injector = Guice.createInjector(new - ReconControllerModule(), + injector = Guice.createInjector(new ReconControllerModule(), new ReconRestServletModule(configuration), new ReconSchemaGenerationModule()); @@ -124,7 +116,6 @@ public Void call() throws Exception { "Initializing certificate."); initializeCertificateClient(configuration); } - reconStorage.persistCurrentState(); } catch (Exception e) { LOG.error("Error during initializing Recon certificate", e); } @@ -172,9 +163,9 @@ public Void call() throws Exception { private void initializeCertificateClient(OzoneConfiguration conf) throws IOException { LOG.info("Initializing secure Recon."); - certClient = new ReconCertificateClient( - new SecurityConfig(configuration), - reconStorage.getReconCertSerialId()); + certClient = new ReconCertificateClient(new SecurityConfig(configuration), + reconStorage.getReconCertSerialId(), reconStorage.getClusterID(), + reconStorage.getReconId(), this::saveNewCertId, null); CertificateClient.InitResponse response = certClient.init(); if (response.equals(CertificateClient.InitResponse.REINIT)) { @@ -182,7 +173,8 @@ private void initializeCertificateClient(OzoneConfiguration conf) reconStorage.unsetReconCertSerialId(); reconStorage.persistCurrentState(); certClient = new ReconCertificateClient(new SecurityConfig(configuration), - reconStorage.getReconCertSerialId()); + reconStorage.getReconCertSerialId(), reconStorage.getClusterID(), + reconStorage.getReconId(), this::saveNewCertId, this::terminateRecon); response = certClient.init(); } LOG.info("Init response: {}", response); @@ -191,7 +183,12 @@ private void initializeCertificateClient(OzoneConfiguration conf) LOG.info("Initialization successful, case:{}.", response); break; case GETCERT: - getSCMSignedCert(conf); + String certId = certClient.signAndStoreCertificate( + certClient.getCSRBuilder().build()); + reconStorage.setReconCertSerialId(certId); + reconStorage.persistCurrentState(); + // set new certificate ID + certClient.setCertificateId(certId); LOG.info("Successfully stored SCM signed certificate, case:{}.", response); break; @@ -209,53 +206,23 @@ private void initializeCertificateClient(OzoneConfiguration conf) } } - /** - * Get SCM signed certificate and store it using certificate client. - * @param config - * */ - private void getSCMSignedCert(OzoneConfiguration config) { + public void saveNewCertId(String newCertId) { try { - PKCS10CertificationRequest csr = ReconUtils.getCSR(config, certClient); - LOG.info("Creating CSR for Recon."); - - SCMSecurityProtocolClientSideTranslatorPB secureScmClient = - HddsServerUtil.getScmSecurityClientWithMaxRetry(config); - HddsProtos.NodeDetailsProto.Builder reconDetailsProtoBuilder = - HddsProtos.NodeDetailsProto.newBuilder() - .setHostName(InetAddress.getLocalHost().getHostName()) - .setClusterId(reconStorage.getClusterID()) - .setUuid(reconStorage.getReconId()) - .setNodeType(HddsProtos.NodeType.RECON); - - SCMSecurityProtocolProtos.SCMGetCertResponseProto response = - secureScmClient.getCertificateChain( - reconDetailsProtoBuilder.build(), - getEncodedString(csr)); - // Persist certificates. - if (response.hasX509CACertificate()) { - String pemEncodedCert = response.getX509Certificate(); - certClient.storeCertificate(pemEncodedCert, true); - certClient.storeCertificate(response.getX509CACertificate(), true, - true); - - // Store Root CA certificate. - if (response.hasX509RootCACertificate()) { - certClient.storeRootCACertificate( - response.getX509RootCACertificate(), true); - } - String reconCertSerialId = getX509Certificate(pemEncodedCert). - getSerialNumber().toString(); - reconStorage.setReconCertSerialId(reconCertSerialId); - } else { - throw new RuntimeException("Unable to retrieve recon certificate " + - "chain"); - } - } catch (IOException | CertificateException e) { - LOG.error("Error while storing SCM signed certificate.", e); - throw new RuntimeException(e); + reconStorage.setReconCertSerialId(newCertId); + reconStorage.persistCurrentState(); + } catch (IOException ex) { + // New cert ID cannot be persisted into VERSION file. + LOG.error("Failed to persist new cert ID {} to VERSION file." + + "Terminating OzoneManager...", newCertId, ex); + terminateRecon(); } } + public void terminateRecon() { + stop(); + terminate(1); + } + /** * Need a way to restart services from tests. */ @@ -278,17 +245,26 @@ public void start() throws Exception { } } - public void stop() throws Exception { + public void stop() { if (isStarted) { LOG.info("Stopping Recon server"); if (httpServer != null) { - httpServer.stop(); + try { + httpServer.stop(); + } catch (Exception e) { + LOG.error("Stopping HttpServer is failed.", e); + } } + if (reconStorageContainerManager != null) { reconStorageContainerManager.stop(); } if (ozoneManagerServiceProvider != null) { - ozoneManagerServiceProvider.stop(); + try { + ozoneManagerServiceProvider.stop(); + } catch (Exception e) { + LOG.error("Stopping ozoneManagerServiceProvider is failed.", e); + } } if (reconTaskStatusMetrics != null) { reconTaskStatusMetrics.unregister(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java index d4ceaec89ff2..bb94c6d29842 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/StorageContainerServiceProviderImpl.java @@ -201,7 +201,8 @@ connectionFactory, getScmDBSnapshotUrl(), try (SCMSnapshotDownloader downloadClient = new InterSCMGrpcClient( hostAddress, grpcPort, configuration, new ReconCertificateClient(new SecurityConfig(configuration), - reconStorage.getReconCertSerialId()))) { + reconStorage.getReconCertSerialId(), + reconStorage.getClusterID(), reconStorage.getReconId()))) { downloadClient.download(targetFile.toPath()).get(); } catch (ExecutionException | InterruptedException e) { LOG.error("Rocks DB checkpoint downloading failed", e);