diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java index 945ca91a4088..bd96f0c26a07 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java @@ -194,6 +194,24 @@ public final class HddsConfigKeys { public static final String HDDS_X509_RENEW_GRACE_DURATION_DEFAULT = "P28D"; + public static final String HDDS_X509_ROOTCA_CERTIFICATE_FILE = + "hdds.x509.rootca.certificate.file"; + + public static final String HDDS_X509_ROOTCA_CERTIFICATE_FILE_DEFAULT = + ""; + + public static final String HDDS_X509_ROOTCA_PUBLIC_KEY_FILE = + "hdds.x509.rootca.public.key.file"; + + public static final String HDDS_X509_ROOTCA_PUBLIC_KEY_FILE_DEFAULT = + ""; + + public static final String HDDS_X509_ROOTCA_PRIVATE_KEY_FILE = + "hdds.x509.rootca.private.key.file"; + + public static final String HDDS_X509_ROOTCA_PRIVATE_KEY_FILE_DEFAULT = + ""; + /** * Do not instantiate. */ diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java index 94401e5e3223..ecc92debdf95 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java @@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.OzoneConfigKeys; import com.google.common.base.Preconditions; + import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED; @@ -37,6 +38,12 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_ALGORITHM; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_FILE; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_FILE_DEFAULT; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_PRIVATE_KEY_FILE; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_PRIVATE_KEY_FILE_DEFAULT; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_PUBLIC_KEY_FILE; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_ROOTCA_PUBLIC_KEY_FILE_DEFAULT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_ENABLED; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_ENABLED_DEFAULT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_PROVIDER; @@ -74,6 +81,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECURITY_SSL_TRUSTSTORE_RELOAD_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; + import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslProvider; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.slf4j.Logger; @@ -111,6 +119,9 @@ public class SecurityConfig { private boolean grpcTlsUseTestCert; private final long keystoreReloadInterval; private final long truststoreReloadInterval; + private final String externalRootCaPublicKeyPath; + private final String externalRootCaPrivateKeyPath; + private final String externalRootCaCert; /** * Constructs a SecurityConfig. @@ -182,8 +193,18 @@ public SecurityConfig(ConfigurationSource configuration) { "greater than maximum Certificate duration"); } + this.externalRootCaCert = this.configuration.get( + HDDS_X509_ROOTCA_CERTIFICATE_FILE, + HDDS_X509_ROOTCA_CERTIFICATE_FILE_DEFAULT); + this.externalRootCaPublicKeyPath = this.configuration.get( + HDDS_X509_ROOTCA_PUBLIC_KEY_FILE, + HDDS_X509_ROOTCA_PUBLIC_KEY_FILE_DEFAULT); + this.externalRootCaPrivateKeyPath = this.configuration.get( + HDDS_X509_ROOTCA_PRIVATE_KEY_FILE, + HDDS_X509_ROOTCA_PRIVATE_KEY_FILE_DEFAULT); + this.crlName = this.configuration.get(HDDS_X509_CRL_NAME, - HDDS_X509_CRL_NAME_DEFAULT); + HDDS_X509_CRL_NAME_DEFAULT); // First Startup -- if the provider is null, check for the provider. if (SecurityConfig.provider == null) { @@ -399,6 +420,18 @@ public SslProvider getGrpcSslProvider() { HDDS_GRPC_TLS_PROVIDER_DEFAULT)); } + public String getExternalRootCaPrivateKeyPath() { + return externalRootCaPrivateKeyPath; + } + + public String getExternalRootCaPublicKeyPath() { + return externalRootCaPublicKeyPath; + } + + public String getExternalRootCaCert() { + return externalRootCaCert; + } + /** * Return true if using test certificates with authority as localhost. This * should be used only for unit test where certificates are generated by diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index fe11a3b72e18..32b446f208d3 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -2087,7 +2087,8 @@ Max time for which certificate issued by SCM CA are valid. This duration is used for self-signed root cert and scm sub-ca certs issued by root ca. The formats accepted are based on the ISO-8601 - duration format PnDTnHnMn.nS + duration format PnDTnHnMn.nS + hdds.x509.signature.algorithm @@ -2095,6 +2096,41 @@ OZONE, HDDS, SECURITY X509 signature certificate. + + hdds.x509.rootca.certificate.file + + Path to an external CA certificate. The file format is expected + to be pem. This certificate is used when initializing SCM to create + a root certificate authority. By default, a self-signed certificate is + generated instead. Note that this certificate is only used for Ozone's + internal communication, and it does not affect the certificates used for + HTTPS protocol at WebUIs as they can be configured separately. + + + + hdds.x509.rootca.private.key.file + + Path to an external private key. The file format is expected + to be pem. This private key is later used when initializing SCM to sign + certificates as the root certificate authority. When not specified a + private and public key is generated instead. + These keys are only used for Ozone's internal communication, and it does + not affect the HTTPS protocol at WebUIs as they can be configured + separately. + + + + hdds.x509.rootca.public.key.file + + Path to an external public key. The file format is expected + to be pem. This public key is later used when initializing SCM to sign + certificates as the root certificate authority. + When only the private key is specified the public key is read from the + external certificate. Note that this is only used for Ozone's internal + communication, and it does not affect the HTTPS protocol at WebUIs as + they can be configured separately. + + ozone.scm.security.handler.count.key 2 diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java index 454ac6c2f4f4..5abd72cabc0c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/DefaultCAServer.java @@ -49,6 +49,8 @@ import java.security.KeyPair; import java.security.NoSuchAlgorithmException; import java.security.NoSuchProviderException; +import java.security.PrivateKey; +import java.security.PublicKey; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.security.spec.InvalidKeySpecException; @@ -252,7 +254,6 @@ public Future requestCertificate( LOG.error("Certificate storage failed, retrying one more time.", e); xcert = signAndStoreCertificate(beginDate, endDate, csr, role); } - xcertHolder.complete(xcert); break; default: @@ -474,19 +475,7 @@ Consumer processVerificationStatus( break; case INITIALIZE: if (type == CAType.SELF_SIGNED_CA) { - consumer = (arg) -> { - try { - generateSelfSignedCA(arg); - } catch (NoSuchProviderException | NoSuchAlgorithmException - | IOException e) { - LOG.error("Unable to initialize CertificateServer.", e); - } - VerificationStatus newStatus = verifySelfSignedCA(arg); - if (newStatus != VerificationStatus.SUCCESS) { - LOG.error("Unable to initialize CertificateServer, failed in " + - "verification."); - } - }; + consumer = this::initRootCa; } else if (type == CAType.INTERMEDIARY_CA) { // For sub CA certificates are generated during bootstrap/init. If // both keys/certs are missing, init/bootstrap is missed to be @@ -506,6 +495,29 @@ Consumer processVerificationStatus( return consumer; } + private void initRootCa(SecurityConfig securityConfig) { + if (isExternalCaSpecified(securityConfig)) { + initWithExternalRootCa(securityConfig); + } else { + try { + generateSelfSignedCA(securityConfig); + } catch (NoSuchProviderException | NoSuchAlgorithmException + | IOException e) { + LOG.error("Unable to initialize CertificateServer.", e); + } + } + VerificationStatus newStatus = verifySelfSignedCA(securityConfig); + if (newStatus != VerificationStatus.SUCCESS) { + LOG.error("Unable to initialize CertificateServer, failed in " + + "verification."); + } + } + + private boolean isExternalCaSpecified(SecurityConfig conf) { + return !conf.getExternalRootCaCert().isEmpty() && + !conf.getExternalRootCaPrivateKeyPath().isEmpty(); + } + /** * Generates a KeyPair for the Certificate. * @@ -529,12 +541,13 @@ private KeyPair generateKeys(SecurityConfig securityConfig) * Generates a self-signed Root Certificate for CA. * * @param securityConfig - SecurityConfig - * @param key - KeyPair. + * @param key - KeyPair. * @throws IOException - on Error. * @throws SCMSecurityException - on Error. */ - private void generateRootCertificate(SecurityConfig securityConfig, - KeyPair key) throws IOException, SCMSecurityException { + private void generateRootCertificate( + SecurityConfig securityConfig, KeyPair key) + throws IOException, SCMSecurityException { Preconditions.checkNotNull(this.config); LocalDateTime beginDate = LocalDateTime.of(LocalDate.now(), LocalTime.MIDNIGHT); @@ -563,7 +576,7 @@ private void generateRootCertificate(SecurityConfig securityConfig, } catch (IOException e) { throw new org.apache.hadoop.hdds.security.x509 .exceptions.CertificateException( - "Error while adding ip to CA self signed certificate", e, + "Error while adding ip to CA self signed certificate", e, CSR_ERROR); } X509CertificateHolder selfSignedCertificate = builder.build(); @@ -573,6 +586,65 @@ private void generateRootCertificate(SecurityConfig securityConfig, certCodec.writeCertificate(selfSignedCertificate); } + private void initWithExternalRootCa(SecurityConfig conf) { + String externalRootCaLocation = conf.getExternalRootCaCert(); + Path extCertPath = Paths.get(externalRootCaLocation); + Path extPrivateKeyPath = Paths.get(conf.getExternalRootCaPrivateKeyPath()); + String externalPublicKeyLocation = conf.getExternalRootCaPublicKeyPath(); + + KeyCodec keyCodec = new KeyCodec(config, componentName); + CertificateCodec certificateCodec = + new CertificateCodec(config, componentName); + try { + Path extCertParent = extCertPath.getParent(); + Path extCertName = extCertPath.getFileName(); + if (extCertParent == null || extCertName == null) { + throw new IOException("External cert path is not correct: " + + extCertPath); + } + X509CertificateHolder certHolder = certificateCodec.readCertificate( + extCertParent, extCertName.toString()); + Path extPrivateKeyParent = extPrivateKeyPath.getParent(); + Path extPrivateKeyFileName = extPrivateKeyPath.getFileName(); + if (extPrivateKeyParent == null || extPrivateKeyFileName == null) { + throw new IOException("External private key path is not correct: " + + extPrivateKeyPath); + } + PrivateKey privateKey = keyCodec.readPrivateKey(extPrivateKeyParent, + extPrivateKeyFileName.toString()); + PublicKey publicKey; + publicKey = readPublicKeyWithExternalData( + externalPublicKeyLocation, keyCodec, certHolder); + keyCodec.writeKey(new KeyPair(publicKey, privateKey)); + certificateCodec.writeCertificate(certHolder); + } catch (IOException | CertificateException | NoSuchAlgorithmException | + InvalidKeySpecException e) { + LOG.error("External root CA certificate initialization failed", e); + } + } + + private PublicKey readPublicKeyWithExternalData( + String externalPublicKeyLocation, KeyCodec keyCodec, + X509CertificateHolder certHolder) + throws CertificateException, NoSuchAlgorithmException, + InvalidKeySpecException, IOException { + PublicKey publicKey; + if (externalPublicKeyLocation.isEmpty()) { + publicKey = CertificateCodec.getX509Certificate(certHolder) + .getPublicKey(); + } else { + Path publicKeyPath = Paths.get(externalPublicKeyLocation); + Path publicKeyPathFileName = publicKeyPath.getFileName(); + Path publicKeyParent = publicKeyPath.getParent(); + if (publicKeyPathFileName == null || publicKeyParent == null) { + throw new IOException("Public key path incorrect: " + publicKeyParent); + } + publicKey = keyCodec.readPublicKey( + publicKeyParent, publicKeyPathFileName.toString()); + } + return publicKey; + } + /** * This represents the verification status of the CA. Based on this enum * appropriate action is taken in the Init. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java index e57510c9ffa1..aead3582249e 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/keys/KeyCodec.java @@ -323,9 +323,9 @@ private synchronized void writeKey(Path basePath, KeyPair keyPair, checkPreconditions(basePath); File privateKeyFile = - Paths.get(location.toString(), privateKeyFileName).toFile(); + Paths.get(basePath.toString(), privateKeyFileName).toFile(); File publicKeyFile = - Paths.get(location.toString(), publicKeyFileName).toFile(); + Paths.get(basePath.toString(), publicKeyFileName).toFile(); checkKeyFile(privateKeyFile, force, publicKeyFile); try (PemWriter privateKeyWriter = new PemWriter(new diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java index 7c3e035f1c67..b049a6a07655 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/certificate/authority/TestDefaultCAServer.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdds.security.x509.certificate.authority; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.validator.routines.DomainValidator; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.exception.SCMSecurityException; @@ -29,7 +30,11 @@ import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; +import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate; import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator; +import org.apache.hadoop.hdds.security.x509.keys.KeyCodec; +import org.apache.hadoop.ozone.OzoneSecurityUtil; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.ozone.test.LambdaTestUtils; import org.bouncycastle.asn1.x509.CRLReason; @@ -50,12 +55,14 @@ import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.function.Consumer; @@ -65,6 +72,7 @@ import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType.SCM; import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer.CAType.INTERMEDIARY_CA; import static org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer.CAType.SELF_SIGNED_CA; +import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateException.ErrorCode.CSR_ERROR; import static org.apache.hadoop.ozone.OzoneConsts.SCM_CA_CERT_STORAGE_DIR; import static org.apache.hadoop.ozone.OzoneConsts.SCM_CA_PATH; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -77,11 +85,12 @@ * Tests the Default CA Server. */ public class TestDefaultCAServer { - private static OzoneConfiguration conf = new OzoneConfiguration(); + private OzoneConfiguration conf; private MockCAStore caStore; @BeforeEach public void init(@TempDir Path tempDir) throws IOException { + conf = new OzoneConfiguration(); conf.set(OZONE_METADATA_DIRS, tempDir.toString()); caStore = new MockCAStore(); } @@ -339,6 +348,57 @@ public void testIntermediaryCAWithEmpty() { () -> scmCA.init(new SecurityConfig(conf), INTERMEDIARY_CA)); } + @Test + public void testExternalRootCA(@TempDir Path tempDir) throws Exception { + //Given an external certificate + String externalCaCertFileName = "CaCert.pem"; + setExternalPathsInConfig(tempDir, externalCaCertFileName); + + SecurityConfig securityConfig = new SecurityConfig(conf); + SCMCertificateClient scmCertificateClient = + new SCMCertificateClient(new SecurityConfig(conf)); + + KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA"); + KeyCodec keyPEMWriter = new KeyCodec(securityConfig, + scmCertificateClient.getComponentName()); + + keyPEMWriter.writeKey(tempDir, keyPair, true); + X509CertificateHolder externalCert = generateExternalCert(keyPair); + + CertificateCodec certificateCodec = new CertificateCodec(securityConfig, + scmCertificateClient.getComponentName()); + + certificateCodec.writeCertificate(tempDir, externalCaCertFileName, + CertificateCodec.getPEMEncodedString(externalCert), true); + + CertificateServer testCA = new DefaultCAServer("testCA", + RandomStringUtils.randomAlphabetic(4), + RandomStringUtils.randomAlphabetic(4), caStore, + new DefaultProfile(), + Paths.get(SCM_CA_CERT_STORAGE_DIR, SCM_CA_PATH).toString()); + //When initializing a CA server with external cert + testCA.init(securityConfig, SELF_SIGNED_CA); + //Then the external cert is set as CA cert for the server. + assertEquals(externalCert, testCA.getCACertificate()); + } + + private void setExternalPathsInConfig(Path tempDir, + String externalCaCertFileName) { + String externalCaCertPart = Paths.get(tempDir.toString(), + externalCaCertFileName).toString(); + String privateKeyPath = Paths.get(tempDir.toString(), + HddsConfigKeys.HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT).toString(); + String publicKeyPath = Paths.get(tempDir.toString(), + HddsConfigKeys.HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT).toString(); + + conf.set(HddsConfigKeys.HDDS_X509_ROOTCA_CERTIFICATE_FILE, + externalCaCertPart); + conf.set(HddsConfigKeys.HDDS_X509_ROOTCA_PRIVATE_KEY_FILE, + privateKeyPath); + conf.set(HddsConfigKeys.HDDS_X509_ROOTCA_PUBLIC_KEY_FILE, + publicKeyPath); + } + @Test public void testIntermediaryCA() throws Exception { @@ -414,4 +474,41 @@ clusterId, scmId, caStore, new DefaultProfile(), } + private X509CertificateHolder generateExternalCert(KeyPair keyPair) + throws Exception { + LocalDateTime notBefore = LocalDateTime.now(); + LocalDateTime notAfter = notBefore.plusYears(1); + String clusterID = UUID.randomUUID().toString(); + String scmID = UUID.randomUUID().toString(); + String subject = "testRootCert"; + + SelfSignedCertificate.Builder builder = + SelfSignedCertificate.newBuilder() + .setBeginDate(notBefore) + .setEndDate(notAfter) + .setClusterID(clusterID) + .setScmID(scmID) + .setSubject(subject) + .setKey(keyPair) + .setConfiguration(conf) + .makeCA(); + + try { + DomainValidator validator = DomainValidator.getInstance(); + // Add all valid ips. + OzoneSecurityUtil.getValidInetsForCurrentHost().forEach( + ip -> { + builder.addIpAddress(ip.getHostAddress()); + if (validator.isValid(ip.getCanonicalHostName())) { + builder.addDnsName(ip.getCanonicalHostName()); + } + }); + } catch (IOException e) { + throw new org.apache.hadoop.hdds.security.x509 + .exceptions.CertificateException( + "Error while adding ip to CA self signed certificate", e, + CSR_ERROR); + } + return builder.build(); + } }