From 48f5214568653244c37d6fd856fc12f045958292 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Mon, 16 Jan 2023 11:26:41 +0800 Subject: [PATCH 1/7] HDDS-7723. Refresh Keys and Certificate used in OzoneSecretManager after certificate renewed --- .../apache/hadoop/hdds/HddsConfigKeys.java | 3 + .../hadoop/hdds/security/OzoneSecretKey.java | 9 +- .../hdds/security/OzoneSecretManager.java | 24 +- .../ssl/PemFileBasedKeyStoresFactory.java | 2 +- .../client/DefaultCertificateClient.java | 20 +- .../hadoop/hdds/utils/HddsServerUtil.java | 6 +- .../ozone/security/OzoneTokenIdentifier.java | 3 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 242 +++++++++++++++++- .../client/CertificateClientTestImpl.java | 30 ++- .../apache/hadoop/ozone/om/OzoneManager.java | 22 +- .../OzoneDelegationTokenSecretManager.java | 14 +- 11 files changed, 338 insertions(+), 37 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java index cb258dfa74dc..705fd93144e0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java @@ -195,6 +195,9 @@ public final class HddsConfigKeys { public static final String HDDS_X509_RENEW_GRACE_DURATION_DEFAULT = "P28D"; public static final String HDDS_NEW_KEY_CERT_DIR_NAME_SUFFIX = "-next"; public static final String HDDS_BACKUP_KEY_CERT_DIR_NAME_SUFFIX = "-previous"; + public static final String HDDS_X509_RENEW_SCAN_INTERVAL = + "hdds.x509.renew.scan.interval"; + public static final String HDDS_X509_RENEW_SCAN_INTERVAL_DEFAULT = "60s"; public static final String HDDS_CONTAINER_REPLICATION_COMPRESSION = "hdds.container.replication.compression"; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretKey.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretKey.java index 0dd5fd4d289f..833a4668b342 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretKey.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretKey.java @@ -49,13 +49,16 @@ public class OzoneSecretKey implements Writable { private PrivateKey privateKey; private PublicKey publicKey; private SecurityConfig securityConfig; + private String certSerialId; - public OzoneSecretKey(int keyId, long expiryDate, KeyPair keyPair) { + public OzoneSecretKey(int keyId, long expiryDate, KeyPair keyPair, + String certificateSerialId) { Preconditions.checkNotNull(keyId); this.keyId = keyId; this.expiryDate = expiryDate; this.privateKey = keyPair.getPrivate(); this.publicKey = keyPair.getPublic(); + this.certSerialId = certificateSerialId; } /* @@ -89,6 +92,10 @@ public PublicKey getPublicKey() { return publicKey; } + public String getCertSerialId() { + return certSerialId; + } + public byte[] getEncodedPrivateKey() { return privateKey.getEncoded(); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java index 575dc7e39169..935d9e0ea953 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java @@ -36,7 +36,9 @@ import java.security.PrivateKey; import java.security.Signature; import java.security.SignatureException; +import java.security.cert.X509Certificate; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** * SecretManager for Ozone Master. Responsible for signing identifiers with @@ -57,7 +59,7 @@ public abstract class OzoneSecretManager private final Text service; private CertificateClient certClient; private volatile boolean running; - private OzoneSecretKey currentKey; + private AtomicReference currentKey; private AtomicInteger currentKeyId; private AtomicInteger tokenSequenceNumber; @@ -112,12 +114,12 @@ public byte[] createPassword(byte[] identifier, PrivateKey privateKey) public byte[] createPassword(T identifier) { if (logger.isDebugEnabled()) { logger.debug("Creating password for identifier: {}, currentKey: {}", - formatTokenId(identifier), currentKey.getKeyId()); + formatTokenId(identifier), currentKey.get().getKeyId()); } byte[] password = null; try { password = createPassword(identifier.getBytes(), - currentKey.getPrivateKey()); + currentKey.get().getPrivateKey()); } catch (IOException ioe) { logger.error("Could not store token {}!!", formatTokenId(identifier), ioe); @@ -166,16 +168,18 @@ public int incrementDelegationTokenSeqNum() { * Update the current master key. This is called once by start method before * tokenRemoverThread is created, */ - private OzoneSecretKey updateCurrentKey(KeyPair keyPair) throws IOException { + private OzoneSecretKey updateCurrentKey(KeyPair keyPair, + X509Certificate certificate) throws IOException { logger.info("Updating the current master key for generating tokens"); // TODO: fix me based on the certificate expire time to set the key // expire time. int newCurrentId = incrementCurrentKeyId(); - OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, -1, - keyPair); - currentKey = newKey; - return currentKey; + OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, + certificate.getNotAfter().getTime(), keyPair, + certificate.getSerialNumber().toString()); + currentKey.set(newKey); + return newKey; } public String formatTokenId(T id) { @@ -193,7 +197,7 @@ public synchronized void start(CertificateClient client) Preconditions.checkState(!isRunning()); setCertClient(client); updateCurrentKey(new KeyPair(certClient.getPublicKey(), - certClient.getPrivateKey())); + certClient.getPrivateKey()), certClient.getCertificate()); setIsRunning(true); } @@ -236,7 +240,7 @@ public void setIsRunning(boolean val) { } public OzoneSecretKey getCurrentKey() { - return currentKey; + return currentKey.get(); } public AtomicInteger getCurrentKeyId() { diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java index 1a8b33f403b4..0d3cb7293846 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/ssl/PemFileBasedKeyStoresFactory.java @@ -141,7 +141,7 @@ private void createKeyManagers(Mode mode) throws public synchronized void init(Mode mode, boolean requireClientAuth) throws IOException, GeneralSecurityException { - monitoringTimer = new Timer(caClient.getComponentName() + "-" + monitoringTimer = new Timer(caClient.getComponentName() + "-" + mode + "-" + SSL_MONITORING_THREAD_NAME, true); // key manager diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index 8647c324f54d..2da4e3950f43 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@ -95,6 +95,7 @@ import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateException.ErrorCode.ROLLBACK_ERROR; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClientWithMaxRetry; +import org.apache.hadoop.security.UserGroupInformation; import org.bouncycastle.cert.X509CertificateHolder; import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; @@ -108,10 +109,10 @@ public abstract class DefaultCertificateClient implements CertificateClient { private static final Random RANDOM = new SecureRandom(); - private static final String CERT_FILE_NAME_FORMAT = "%s.crt"; - private static final String CA_CERT_PREFIX = "CA-"; + public static final String CERT_FILE_NAME_FORMAT = "%s.crt"; + public static final String CA_CERT_PREFIX = "CA-"; private static final int CA_CERT_PREFIX_LEN = 3; - private static final String ROOT_CA_CERT_PREFIX = "ROOTCA-"; + public static final String ROOT_CA_CERT_PREFIX = "ROOTCA-"; private static final int ROOT_CA_PREFIX_LEN = 7; private final Logger logger; private final SecurityConfig securityConfig; @@ -142,6 +143,7 @@ public abstract class DefaultCertificateClient implements CertificateClient { private Consumer certIdSaveCallback; private Runnable shutdownCallback; private SCMSecurityProtocolClientSideTranslatorPB scmSecurityProtocolClient; + private static UserGroupInformation ugi; DefaultCertificateClient(SecurityConfig securityConfig, Logger log, String certSerialId, String component, @@ -1361,7 +1363,7 @@ public SCMSecurityProtocolClientSideTranslatorPB getScmSecureClient() if (scmSecurityProtocolClient == null) { scmSecurityProtocolClient = getScmSecurityClientWithMaxRetry( - (OzoneConfiguration) securityConfig.getConfiguration()); + (OzoneConfiguration) securityConfig.getConfiguration(), ugi); } return scmSecurityProtocolClient; } @@ -1372,6 +1374,11 @@ public void setSecureScmClient( scmSecurityProtocolClient = client; } + @VisibleForTesting + public static void setUgi(UserGroupInformation user) { + ugi = user; + } + public synchronized void startCertificateMonitor() { Preconditions.checkNotNull(getCertificate(), "Component certificate should not be empty"); @@ -1390,8 +1397,9 @@ public synchronized void startCertificateMonitor() { } this.executorService.scheduleAtFixedRate(new CertificateLifetimeMonitor(), timeBeforeGracePeriod, interval, TimeUnit.MILLISECONDS); - getLogger().info("CertificateLifetimeMonitor is started with first delay" + - " {} ms and interval {} ms.", timeBeforeGracePeriod, interval); + getLogger().info("CertificateLifetimeMonitor for {} is started with " + + "first delay {} ms and interval {} ms.", component, + timeBeforeGracePeriod, interval); } /** diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java index 80970ff1bf65..af3c5d5ed475 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java @@ -450,8 +450,8 @@ public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient( } public static SCMSecurityProtocolClientSideTranslatorPB - getScmSecurityClientWithMaxRetry(OzoneConfiguration conf) - throws IOException { + getScmSecurityClientWithMaxRetry(OzoneConfiguration conf, + UserGroupInformation ugi) throws IOException { // Certificate from SCM is required for DN startup to succeed, so retry // for ever. In this way DN start up is resilient to SCM service running // status. @@ -464,7 +464,7 @@ public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient( return new SCMSecurityProtocolClientSideTranslatorPB( new SCMSecurityProtocolFailoverProxyProvider(configuration, - UserGroupInformation.getCurrentUser())); + ugi == null ? UserGroupInformation.getCurrentUser() : ugi)); } public static SCMSecurityProtocolClientSideTranslatorPB diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java index 01fcaf893237..19f3e7c4a25c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/security/OzoneTokenIdentifier.java @@ -382,7 +382,8 @@ public String toString() { .append(", strToSign=").append(getStrToSign()) .append(", signature=").append(getSignature()) .append(", awsAccessKeyId=").append(getAwsAccessId()) - .append(", omServiceId=").append(getOmServiceId()); + .append(", omServiceId=").append(getOmServiceId()) + .append(", omCertSerialId=").append(getOmCertSerialId()); return buffer.toString(); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index 5952ce5947cc..95f07d7d3102 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -29,8 +29,11 @@ import java.time.Duration; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.ZoneId; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Date; +import java.util.List; import java.util.Properties; import java.util.UUID; import java.util.concurrent.Callable; @@ -56,8 +59,14 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.security.exception.SCMSecurityException; import org.apache.hadoop.hdds.security.x509.SecurityConfig; +import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultApprover; +import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile; import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer; +import org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; +import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate; import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator; @@ -95,6 +104,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION_DEFAULT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY; @@ -161,7 +171,7 @@ @InterfaceAudience.Private public final class TestSecureOzoneCluster { - private static final String COMPONENT = "test"; + private static final String COMPONENT = "om"; private static final String OM_CERT_SERIAL_ID = "9879877970576"; private static final Logger LOG = LoggerFactory .getLogger(TestSecureOzoneCluster.class); @@ -187,6 +197,8 @@ public final class TestSecureOzoneCluster { private String scmId; private String omId; private OzoneManagerProtocolClientSideTranslatorPB omClient; + private KeyPair keyPair; + private Path omMetaDirPath; @Before public void init() { @@ -211,8 +223,8 @@ public void init() { DefaultMetricsSystem.setMiniClusterMode(true); ExitUtils.disableSystemExit(); final String path = folder.newFolder().toString(); - Path metaDirPath = Paths.get(path, "om-meta"); - conf.set(OZONE_METADATA_DIRS, metaDirPath.toString()); + omMetaDirPath = Paths.get(path, "om-meta"); + conf.set(OZONE_METADATA_DIRS, omMetaDirPath.toString()); conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name()); conf.set(HDDS_X509_RENEW_GRACE_DURATION, "PT5S"); // 5s @@ -544,7 +556,7 @@ public void testAccessControlExceptionOnClient() throws Exception { private void generateKeyPair() throws Exception { HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(conf); - KeyPair keyPair = keyGenerator.generateKey(); + keyPair = keyGenerator.generateKey(); KeyCodec pemWriter = new KeyCodec(new SecurityConfig(conf), COMPONENT); pemWriter.writeKey(keyPair, true); } @@ -1058,6 +1070,196 @@ public void testCertificateRotationUnRecoverableFailure() throws Exception { 1000, certificateLifetime * 1000); } + /** + * Tests delegation token renewal after a certificate renew + */ + @Test + public void testDelegationTokenRenewCrossCertificateRenew() throws Exception { + try { + // Setup secure OM for start. + int certLifetime = 15 * 1000; // seconds + OzoneConfiguration newConf = new OzoneConfiguration(conf); + int tokenMaxLifetime = certLifetime * 2; + newConf.setLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, tokenMaxLifetime); + newConf.set(HDDS_X509_DEFAULT_DURATION, + Duration.ofMillis(certLifetime).toString()); + + setupOm(newConf); + OzoneManager.setTestSecureOmFlag(true); + + CertificateClientTestImpl certClient = + new CertificateClientTestImpl(newConf); + X509Certificate omCert = certClient.getCertificate(); + System.out.println("OmCert " + omCert.getSerialNumber().toString()); + // Start OM + om.setCertClient(certClient); + om.start(); + GenericTestUtils.waitFor(() -> om.isLeaderReady(), 500, 5000); + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + + // Get first OM client which will authenticate via Kerberos + omClient = new OzoneManagerProtocolClientSideTranslatorPB( + OmTransportFactory.create(conf, ugi, null), + RandomStringUtils.randomAscii(5)); + + // Since client is already connected get a delegation token + Token token = omClient.getDelegationToken( + new Text("om")); + + // Check if token is of right kind and renewer is running om instance + assertNotNull(token); + assertEquals("OzoneToken", token.getKind().toString()); + assertEquals(OmUtils.getOmRpcAddress(conf), + token.getService().toString()); + + // Renew delegation token + long expiryTime = omClient.renewDelegationToken(token); + assertTrue(expiryTime > 0); + + // Wait for OM certificate to expire and renew + GenericTestUtils.waitFor(() -> omCert.getNotAfter().before(new Date()), + 500, certLifetime); + certClient.renewKey(); + assertFalse(omCert.getSerialNumber().toString().equals( + certClient.getCertificate().getSerialNumber().toString())); + System.out.println("New OmCert " + + certClient.getCertificate().getSerialNumber().toString()); + + // Test delegation renewal after certificate renewed; + expiryTime = omClient.renewDelegationToken(token); + assertTrue(expiryTime > 0); + } finally { + if (om != null) { + om.stop(); + } + IOUtils.closeQuietly(om); + } + } + + /** + * Test functionality to get SCM signed certificate for OM. + */ + @Test + public void testOMGrpcServerCertificateRenew() throws Exception { + initSCM(); + try { + scm = HddsTestUtils.getScmSimple(conf); + scm.start(); + + int certLifetime = 30; // second + om = createOM(certLifetime, conf); + + X509Certificate omCert = om.getCertificateClient().getCertificate(); + X509Certificate caCert = om.getCertificateClient().getCACertificate(); + X509Certificate rootCaCert = + om.getCertificateClient().getRootCACertificate(); + List certList = new ArrayList<>(); + certList.add(caCert); + certList.add(rootCaCert); + // set certificates in GrpcOmTransport + GrpcOmTransport.setCaCerts(certList); + + GenericTestUtils.waitFor(() -> om.isLeaderReady(), 500, 10000); + String transportCls = GrpcOmTransportFactory.class.getName(); + conf.set(OZONE_OM_TRANSPORT_CLASS, transportCls); + OzoneClient client = OzoneClientFactory.getRpcClient(conf); + + ServiceInfoEx serviceInfoEx = client.getObjectStore() + .getClientProxy().getOzoneManagerClient().getServiceInfo(); + Assert.assertTrue(serviceInfoEx.getCaCertificate().equals( + CertificateCodec.getPEMEncodedString(caCert))); + + // Wait for OM certificate to expire and renew + GenericTestUtils.waitFor(() -> omCert.getNotAfter().before(new Date()), + 500, certLifetime * 1000); + + // rerun the command using old client, it should succeed + serviceInfoEx = client.getObjectStore() + .getClientProxy().getOzoneManagerClient().getServiceInfo(); + Assert.assertTrue(serviceInfoEx.getCaCertificate().equals( + CertificateCodec.getPEMEncodedString(caCert))); + client.close(); + + // rerun the command using new client, it should succeed too. + try { + OzoneClientFactory.getRpcClient(conf); + } catch (Exception e) { + fail("Create client should succeed for certificate is renewed"); + } + } finally { + DefaultCertificateClient.setUgi(null); + OzoneManager.setUgi(null); + GrpcOmTransport.setCaCerts(null); + if (scm != null) { + scm.stop(); + } + if (om != null) { + om.stop(); + } + IOUtils.closeQuietly(om); + } + } + + /** + * Create an OzoneManager instance. + * @param certLifetime lifetime in seconds. + * @return + * @throws Exception + */ + private OzoneManager createOM(int certLifetime, OzoneConfiguration config) + throws Exception { + config.set(OZONE_METADATA_DIRS, omMetaDirPath.toString()); + config.set(HDDS_X509_DEFAULT_DURATION, + Duration.ofSeconds(certLifetime).toString()); + config.set(HDDS_SECURITY_SSL_KEYSTORE_RELOAD_INTERVAL, "1s"); + config.set(HDDS_SECURITY_SSL_TRUSTSTORE_RELOAD_INTERVAL, "1s"); + + // initialize OmStorage, save om Cert and CA Certs to disk + OMStorage omStore = new OMStorage(config); + omStore.setClusterId(clusterId); + omStore.setOmId(omId); + SecurityConfig securityConfig = new SecurityConfig(config); + SCMCertificateClient scmCertClient = scm.getScmCertificateClient(); + CertificateCodec certCodec = new CertificateCodec(securityConfig, "om"); + X509Certificate scmCert = scmCertClient.getCertificate(); + X509Certificate rootCert = scmCertClient.getCACertificate(); + X509CertificateHolder certHolder = generateX509CertHolder(config, keyPair, + new KeyPair(scmCertClient.getPublicKey(), + scmCertClient.getPrivateKey()), scmCert, + Duration.ofSeconds(certLifetime)); + String certId = certHolder.getSerialNumber().toString(); + certCodec.writeCertificate(certHolder); + certCodec.writeCertificate(CertificateCodec.getCertificateHolder(scmCert), + String.format(DefaultCertificateClient.CERT_FILE_NAME_FORMAT, + DefaultCertificateClient.CA_CERT_PREFIX + + scmCert.getSerialNumber().toString()), false); + certCodec.writeCertificate(CertificateCodec.getCertificateHolder( + scmCertClient.getCACertificate()), + String.format(DefaultCertificateClient.CERT_FILE_NAME_FORMAT, + DefaultCertificateClient.ROOT_CA_CERT_PREFIX + + rootCert.getSerialNumber().toString()), false); + omStore.setOmCertSerialId(certId); + omStore.initialize(); + + config.setBoolean(HDDS_GRPC_TLS_ENABLED, true); + config.setBoolean(OZONE_OM_S3_GPRC_SERVER_ENABLED, true); + OzoneManager.setTestSecureOmFlag(true); + UserGroupInformation ugi = + UserGroupInformation.loginUserFromKeytabAndReturnUGI( + config.get(OZONE_OM_KERBEROS_PRINCIPAL_KEY), + omKeyTab.getCanonicalPath()); + // In this process, SCM has already login using Kerberos. So pass + // specific UGI to DefaultCertificateClient and OzoneManager to avoid + // conflict with SCM procedure. + DefaultCertificateClient.setUgi(ugi); + OzoneManager.setUgi(ugi); + OzoneManager ozoneManager = OzoneManager.createOm(config); + ozoneManager.start(); + return ozoneManager; + } + +>>>>>>> 624df3bf0 (HDDS-7723. Refresh Keys and Certificate used in OzoneSecretManager after certificate renewed) public void validateCertificate(X509Certificate cert) throws Exception { // Assert that we indeed have a self signed certificate. @@ -1125,4 +1327,36 @@ private static X509CertificateHolder generateX509CertHolder( .setScmID("test") .build(); } + + private static X509CertificateHolder generateX509CertHolder ( + OzoneConfiguration conf, KeyPair keyPair, KeyPair rootKeyPair, + X509Certificate rootCert, Duration certLifetime) throws Exception { + // Generate normal certificate, signed by RootCA certificate + SecurityConfig secConfig = new SecurityConfig(conf); + DefaultApprover approver = new DefaultApprover(new DefaultProfile(), + secConfig); + + CertificateSignRequest.Builder csrBuilder = + new CertificateSignRequest.Builder(); + // Get host name. + csrBuilder.setKey(keyPair) + .setConfiguration(conf) + .setScmID("test") + .setClusterID("cluster") + .setSubject("localhost") + .setDigitalSignature(true) + .setDigitalEncryption(true); + + LocalDateTime start = LocalDateTime.now(); + String certDuration = conf.get(HDDS_X509_DEFAULT_DURATION, + HDDS_X509_DEFAULT_DURATION_DEFAULT); + X509CertificateHolder certificateHolder = + approver.sign(secConfig, rootKeyPair.getPrivate(), + new X509CertificateHolder(rootCert.getEncoded()), + Date.from(start.atZone(ZoneId.systemDefault()).toInstant()), + Date.from(start.plus(Duration.parse(certDuration)) + .atZone(ZoneId.systemDefault()).toInstant()), + csrBuilder.build(), "test", "cluster"); + return certificateHolder; + } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java index aed1a3852233..77dc76eecb87 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java @@ -19,9 +19,14 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.security.InvalidKeyException; import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; import java.security.PrivateKey; import java.security.PublicKey; +import java.security.Signature; +import java.security.SignatureException; import java.security.cert.CertStore; import java.security.cert.X509Certificate; import java.time.Duration; @@ -30,6 +35,8 @@ import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory; @@ -52,6 +59,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION_DEFAULT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION_DEFAULT; +import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateException.ErrorCode.CRYPTO_SIGNATURE_VERIFICATION_ERROR; /** * Test implementation for CertificateClient. To be used only for test @@ -70,6 +78,7 @@ public class CertificateClientTestImpl implements CertificateClient { private DefaultApprover approver; private KeyStoresFactory serverKeyStoresFactory; private KeyStoresFactory clientKeyStoresFactory; + private Map certificateMap; public CertificateClientTestImpl(OzoneConfiguration conf) throws Exception { this(conf, true); @@ -77,6 +86,7 @@ public CertificateClientTestImpl(OzoneConfiguration conf) throws Exception { public CertificateClientTestImpl(OzoneConfiguration conf, boolean rootCA) throws Exception { + certificateMap = new ConcurrentHashMap<>(); securityConfig = new SecurityConfig(conf); keyGen = new HDDSKeyGenerator(securityConfig.getConfiguration()); keyPair = keyGen.generateKey(); @@ -100,6 +110,7 @@ public CertificateClientTestImpl(OzoneConfiguration conf, boolean rootCA) .makeCA(); rootCert = new JcaX509CertificateConverter().getCertificate( builder.build()); + certificateMap.put(rootCert.getSerialNumber().toString(), rootCert); // Generate normal certificate, signed by RootCA certificate approver = new DefaultApprover(new DefaultProfile(), securityConfig); @@ -126,6 +137,8 @@ public CertificateClientTestImpl(OzoneConfiguration conf, boolean rootCA) csrBuilder.build(), "scm1", "cluster1"); x509Certificate = new JcaX509CertificateConverter().getCertificate(certificateHolder); + certificateMap.put(x509Certificate.getSerialNumber().toString(), + x509Certificate); serverKeyStoresFactory = SecurityUtil.getServerKeyStoresFactory( securityConfig, this, true); @@ -152,7 +165,7 @@ public PublicKey getPublicKey() { @Override public X509Certificate getCertificate(String certSerialId) throws CertificateException { - return x509Certificate; + return certificateMap.get(certSerialId); } @Override @@ -194,7 +207,18 @@ public boolean verifySignature(InputStream stream, byte[] signature, @Override public boolean verifySignature(byte[] data, byte[] signature, X509Certificate cert) throws CertificateException { - return true; + try { + Signature sign = Signature.getInstance(getSignatureAlgorithm(), + getSecurityProvider()); + sign.initVerify(cert); + sign.update(data); + return sign.verify(signature); + } catch (NoSuchAlgorithmException | NoSuchProviderException + | InvalidKeyException | SignatureException e) { + System.out.println("Error while signing the stream " + e.getMessage()); + throw new CertificateException("Error while signing the stream", e, + CRYPTO_SIGNATURE_VERIFICATION_ERROR); + } } @Override @@ -346,6 +370,8 @@ public void renewKey() throws Exception { // Save certificate and private key to keyStore keyPair = newKeyPair; x509Certificate = newX509Certificate; + certificateMap.put(x509Certificate.getSerialNumber().toString(), + x509Certificate); System.out.println(new Date() + " certificated is renewed"); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 215f0e5daec0..1f00534f65a3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -31,6 +31,7 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.security.PrivilegedExceptionAction; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -411,6 +412,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl // Test flags private static boolean testReloadConfigFlag = false; private static boolean testSecureOmFlag = false; + private static UserGroupInformation ugi; private final OzoneLockProvider ozoneLockProvider; private OMPerformanceMetrics perfMetrics; @@ -953,6 +955,14 @@ private OzoneDelegationTokenSecretManager createDelegationTokenSecretManager( conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); + Duration certificateGracePeriod = + new SecurityConfig(conf).getRenewalGracePeriod(); + if (certificateGracePeriod.toMillis() < tokenMaxLifetime) { + throw new IllegalArgumentException("Certificate grace period " + + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + + " should be greater than maximum delegation token lifetime " + + OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY); + } return new OzoneDelegationTokenSecretManager.Builder() .setConf(conf) @@ -1214,7 +1224,7 @@ private static boolean isOzoneSecurityEnabled() { private static void loginOMUserIfSecurityEnabled(OzoneConfiguration conf) throws IOException, AuthenticationException { securityEnabled = OzoneSecurityUtil.isSecurityEnabled(conf); - if (securityEnabled) { + if (securityEnabled && ugi == null) { loginOMUser(conf); } } @@ -3943,6 +3953,16 @@ public static void setTestSecureOmFlag(boolean testSecureOmFlag) { OzoneManager.testSecureOmFlag = testSecureOmFlag; } + @VisibleForTesting + public static void setUgi(UserGroupInformation user) { + OzoneManager.ugi = user; + } + + @VisibleForTesting + public static UserGroupInformation getUgi() { + return OzoneManager.ugi; + } + public OMNodeDetails getNodeDetails() { return omNodeDetails; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java index 3e023d1f92ec..742726356f6f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java @@ -214,9 +214,9 @@ public Token createToken(Text owner, Text renewer, Token token = new Token<>(identifier.getBytes(), password, identifier.getKind(), getService()); - if (LOG.isDebugEnabled()) { - LOG.debug("Created delegation token: {}", token); - } + //if (LOG.isDebugEnabled()) { + LOG.info("Created delegation token: {}", token); + //} return token; } @@ -272,11 +272,7 @@ private void updateIdentifierDetails(OzoneTokenIdentifier identifier) { * Get OM certificate serial id. * */ private String getOmCertificateSerialId() { - if (omCertificateSerialId == null) { - omCertificateSerialId = - getCertClient().getCertificate().getSerialNumber().toString(); - } - return omCertificateSerialId; + return getCertClient().getCertificate().getSerialNumber().toString(); } private String getOmServiceId() { @@ -480,6 +476,7 @@ public boolean verifySignature(OzoneTokenIdentifier identifier, try { signerCert.checkValidity(); } catch (CertificateExpiredException | CertificateNotYetValidException e) { + LOG.error("signerCert {} is invalid", signerCert, e); return false; } @@ -487,6 +484,7 @@ public boolean verifySignature(OzoneTokenIdentifier identifier, return getCertClient().verifySignature(identifier.getBytes(), password, signerCert); } catch (CertificateException e) { + LOG.error("verifySignature with signerCert {} failed", signerCert, e); return false; } } From 2a83c9e57376a652dd54013db8d3866adbb07d0d Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Mon, 16 Jan 2023 20:27:36 +0800 Subject: [PATCH 2/7] HDDS-7723. Refresh Keys and Certificate used in OzoneSecretManager after certificate renewed. --- .../ec/reconstruction/TokenHelper.java | 15 +- .../hdds/security/OzoneSecretManager.java | 26 +- .../token/ContainerTokenSecretManager.java | 5 +- .../token/OzoneBlockTokenSecretManager.java | 5 +- .../token/ShortLivedTokenSecretManager.java | 9 +- .../certificate/client/CertificateClient.java | 6 + .../client/CertificateNotification.java | 31 ++ .../client/DefaultCertificateClient.java | 21 +- .../TestOzoneBlockTokenSecretManager.java | 2 +- .../security/token/TokenVerifierTests.java | 2 +- .../security/x509/CertificateClientTest.java | 5 + .../hdds/scm/ha/SCMSnapshotProvider.java | 6 +- .../scm/server/StorageContainerManager.java | 24 +- .../scm/storage/TestContainerCommandsEC.java | 4 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 324 +++++++++++------- .../client/CertificateClientTestImpl.java | 61 +++- .../client/rpc/TestSecureOzoneRpcClient.java | 3 +- .../ozoneimpl/TestOzoneContainerWithTLS.java | 4 +- .../ozoneimpl/TestSecureOzoneContainer.java | 92 ++++- .../server/TestSecureContainerServer.java | 6 +- .../apache/hadoop/ozone/om/OzoneManager.java | 26 +- .../OzoneDelegationTokenSecretManager.java | 9 +- 22 files changed, 489 insertions(+), 197 deletions(-) create mode 100644 hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java index ace44ba9a39e..53fee24bed8b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java @@ -69,11 +69,20 @@ class TokenHelper { HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT, TimeUnit.MILLISECONDS); - String certId = certClient.getCertificate().getSerialNumber().toString(); + long certificateGracePeriod = + conf.getTimeDuration(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT, + TimeUnit.MILLISECONDS); + if (expiryTime > certificateGracePeriod) { + throw new IllegalArgumentException("Certificate grace period " + + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + + " should be greater than maximum block/container token lifetime " + + HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME); + } if (blockTokenEnabled) { blockTokenMgr = new OzoneBlockTokenSecretManager( - securityConfig, expiryTime, certId); + securityConfig, expiryTime); blockTokenMgr.start(certClient); } else { blockTokenMgr = null; @@ -81,7 +90,7 @@ class TokenHelper { if (containerTokenEnabled) { containerTokenMgr = new ContainerTokenSecretManager( - securityConfig, expiryTime, certId); + securityConfig, expiryTime); containerTokenMgr.start(certClient); } else { containerTokenMgr = null; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java index 935d9e0ea953..6c8530684375 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateNotification; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.token.SecretManager; @@ -47,7 +48,7 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public abstract class OzoneSecretManager - extends SecretManager { + extends SecretManager implements CertificateNotification { private final Logger logger; /** @@ -83,6 +84,7 @@ public OzoneSecretManager(SecurityConfig secureConf, long tokenMaxLifetime, tokenSequenceNumber = new AtomicInteger(); this.service = service; this.logger = logger; + this.currentKey = new AtomicReference<>(); } @@ -169,11 +171,9 @@ public int incrementDelegationTokenSeqNum() { * tokenRemoverThread is created, */ private OzoneSecretKey updateCurrentKey(KeyPair keyPair, - X509Certificate certificate) throws IOException { + X509Certificate certificate) { logger.info("Updating the current master key for generating tokens"); - // TODO: fix me based on the certificate expire time to set the key - // expire time. int newCurrentId = incrementCurrentKeyId(); OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, certificate.getNotAfter().getTime(), keyPair, @@ -182,6 +182,19 @@ private OzoneSecretKey updateCurrentKey(KeyPair keyPair, return newKey; } + public void notifyCertificateRenewed(String oldCertId, String newCertId) { + if (!oldCertId.equals(getCertSerialId())) { + logger.info("Old certificate Id doesn't match. Holding {}, oldCertId {}", + getCertSerialId(), oldCertId); + } + if (!newCertId.equals(certClient.getCertificate())) { + logger.info("New certificate Id doesn't match. Holding in caClient {}," + + " newCertId {}", newCertId, certClient.getCertificate()); + } + updateCurrentKey(new KeyPair(certClient.getPublicKey(), + certClient.getPrivateKey()), certClient.getCertificate()); + } + public String formatTokenId(T id) { return "(" + id + ")"; } @@ -198,6 +211,7 @@ public synchronized void start(CertificateClient client) setCertClient(client); updateCurrentKey(new KeyPair(certClient.getPublicKey(), certClient.getPrivateKey()), certClient.getCertificate()); + client.registerNotificationReceiver(this); setIsRunning(true); } @@ -247,6 +261,10 @@ public AtomicInteger getCurrentKeyId() { return currentKeyId; } + public String getCertSerialId() { + return currentKey.get().getCertSerialId(); + } + public AtomicInteger getTokenSequenceNumber() { return tokenSequenceNumber; } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java index 2015c56645e9..e9f37f3de120 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ContainerTokenSecretManager.java @@ -41,9 +41,8 @@ public class ContainerTokenSecretManager private static final Logger LOG = LoggerFactory.getLogger(ContainerTokenSecretManager.class); - public ContainerTokenSecretManager(SecurityConfig conf, - long tokenLifetime, String certSerialId) { - super(conf, tokenLifetime, certSerialId, LOG); + public ContainerTokenSecretManager(SecurityConfig conf, long tokenLifetime) { + super(conf, tokenLifetime, LOG); } public ContainerTokenIdentifier createIdentifier(String user, diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java index 886d6b354ab6..3dc7a395a1b4 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/OzoneBlockTokenSecretManager.java @@ -44,9 +44,8 @@ public class OzoneBlockTokenSecretManager extends private static final Logger LOG = LoggerFactory .getLogger(OzoneBlockTokenSecretManager.class); - public OzoneBlockTokenSecretManager(SecurityConfig conf, - long tokenLifetime, String omCertSerialId) { - super(conf, tokenLifetime, omCertSerialId, LOG); + public OzoneBlockTokenSecretManager(SecurityConfig conf, long tokenLifetime) { + super(conf, tokenLifetime, LOG); } @Override diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java index a40317880916..966cf1cf5f39 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/token/ShortLivedTokenSecretManager.java @@ -39,12 +39,9 @@ private static final Text SERVICE = new Text("HDDS_SERVICE"); - private final String certSerialId; - protected ShortLivedTokenSecretManager(SecurityConfig conf, - long tokenLifetime, String certSerialId, Logger logger) { + long tokenLifetime, Logger logger) { super(conf, tokenLifetime, tokenLifetime, SERVICE, logger); - this.certSerialId = certSerialId; } @Override @@ -95,10 +92,6 @@ protected Instant getTokenExpiryTime() { return Instant.now().plusMillis(getTokenMaxLifetime()); } - protected String getCertSerialId() { - return certSerialId; - } - public Token generateToken(T tokenIdentifier) { return new Token<>(tokenIdentifier.getBytes(), createPassword(tokenIdentifier), tokenIdentifier.getKind(), diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java index 641b26edb1a2..caeca376bf3b 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java @@ -362,4 +362,10 @@ default void assertValidKeysAndCertificate() throws OzoneSecurityException { * Return the store factory for key manager and trust manager for client. */ KeyStoresFactory getClientKeyStoresFactory() throws CertificateException; + + /** + * Register a receiver that will be called after the certificate renewed. + * @param receiver + */ + void registerNotificationReceiver(CertificateNotification receiver); } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java new file mode 100644 index 000000000000..3595cf21a981 --- /dev/null +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.hdds.security.x509.certificate.client; + +public interface CertificateNotification { + /** + * Notify the class implementing this interface that certificate is renewed. + * Note that the new leader can possibly be this server. + * + * @param oldCertId The old cert id before renew. + * @param newCertId The new cert id after renew. + */ + void notifyCertificateRenewed(String oldCertId, String newCertId); +} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index 2da4e3950f43..e5d719c7668f 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@ -46,10 +46,12 @@ import java.time.ZoneId; import java.util.ArrayList; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Random; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -143,6 +145,7 @@ public abstract class DefaultCertificateClient implements CertificateClient { private Consumer certIdSaveCallback; private Runnable shutdownCallback; private SCMSecurityProtocolClientSideTranslatorPB scmSecurityProtocolClient; + private Set notificationReceivers; private static UserGroupInformation ugi; DefaultCertificateClient(SecurityConfig securityConfig, Logger log, @@ -157,6 +160,7 @@ public abstract class DefaultCertificateClient implements CertificateClient { this.component = component; this.certIdSaveCallback = saveCertId; this.shutdownCallback = shutdown; + this.notificationReceivers = new HashSet<>(); loadAllCertificates(); } @@ -1100,6 +1104,17 @@ public KeyStoresFactory getClientKeyStoresFactory() return clientKeyStoresFactory; } + /** + * Register a receiver that will be called after the certificate renewed. + * @param receiver + */ + @Override + public void registerNotificationReceiver(CertificateNotification receiver) { + synchronized (notificationReceivers) { + notificationReceivers.add(receiver); + } + } + @Override public synchronized void close() throws IOException { if (executorService != null) { @@ -1411,7 +1426,8 @@ public void run() { renewLock.lock(); try { - Duration timeLeft = timeBeforeExpiryGracePeriod(getCertificate()); + X509Certificate currentCert = getCertificate(); + Duration timeLeft = timeBeforeExpiryGracePeriod(currentCert); if (timeLeft.isZero()) { String newCertId; try { @@ -1442,6 +1458,9 @@ public void run() { reloadKeyAndCertificate(newCertId); // cleanup backup directory cleanBackupDir(); + // notify notification receivers + notificationReceivers.forEach(r -> r.notifyCertificateRenewed( + currentCert.getSerialNumber().toString(), newCertId)); } } finally { renewLock.unlock(); diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java index ad435a0f30b5..00b8b339e99b 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TestOzoneBlockTokenSecretManager.java @@ -105,7 +105,7 @@ public void setUp() throws Exception { .generateCertificate("CN=OzoneMaster", keyPair, 30, ALGORITHM); omCertSerialId = x509Certificate.getSerialNumber().toString(); secretManager = new OzoneBlockTokenSecretManager(securityConfig, - TimeUnit.HOURS.toMillis(1), omCertSerialId); + TimeUnit.HOURS.toMillis(1)); client = Mockito.mock(DefaultCertificateClient.class); when(client.getCertificate()).thenReturn(x509Certificate); when(client.getCertificate(anyString())). diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java index 2744823cb248..009b7a0105c3 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/token/TokenVerifierTests.java @@ -231,7 +231,7 @@ private static Token anyToken() { private class MockTokenManager extends ShortLivedTokenSecretManager { MockTokenManager(SecurityConfig conf) { - super(conf, TimeUnit.HOURS.toMillis(1), CERT_ID, LOG); + super(conf, TimeUnit.HOURS.toMillis(1), LOG); } @Override diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java index 1c5e1118e0bb..6c4444babe75 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/security/x509/CertificateClientTest.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateNotification; import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.crl.CRLInfo; import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; @@ -253,6 +254,10 @@ public KeyStoresFactory getClientKeyStoresFactory() return null; } + @Override + public void registerNotificationReceiver(CertificateNotification receiver) { + } + public void renewKey() throws Exception { KeyPair newKeyPair = KeyStoreTestUtil.generateKeyPair("RSA"); X509Certificate newCert = KeyStoreTestUtil.generateCertificate( diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java index ac6716c7d45c..c2214635959e 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMSnapshotProvider.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.hdds.utils.db.RocksDBCheckpoint; @@ -58,11 +58,11 @@ public class SCMSnapshotProvider { private Map peerNodesMap; - private final SCMCertificateClient scmCertificateClient; + private final CertificateClient scmCertificateClient; public SCMSnapshotProvider(ConfigurationSource conf, List peerNodes, - SCMCertificateClient scmCertificateClient) { + CertificateClient scmCertificateClient) { LOG.info("Initializing SCM Snapshot Provider"); this.conf = conf; this.scmCertificateClient = scmCertificateClient; diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 1452c7f6b3a5..07a3e87fc473 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore; import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultCAProfile; import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile; +import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.server.OzoneAdmins; @@ -254,7 +255,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl private ReplicationManager replicationManager; private SCMSafeModeManager scmSafeModeManager; - private SCMCertificateClient scmCertificateClient; + private CertificateClient scmCertificateClient; private ContainerTokenSecretManager containerTokenMgr; private JvmPauseMonitor jvmPauseMonitor; @@ -892,10 +893,15 @@ public CertificateServer getScmCertificateServer() { return getSecurityProtocolServer().getScmCertificateServer(); } - public SCMCertificateClient getScmCertificateClient() { + public CertificateClient getScmCertificateClient() { return scmCertificateClient; } + @VisibleForTesting + public void setScmCertificateClient(CertificateClient client) { + scmCertificateClient = client; + } + private ContainerTokenSecretManager createContainerTokenSecretManager( OzoneConfiguration conf) throws IOException { @@ -903,6 +909,16 @@ private ContainerTokenSecretManager createContainerTokenSecretManager( HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT, TimeUnit.MILLISECONDS); + long certificateGracePeriod = + conf.getTimeDuration(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT, + TimeUnit.MILLISECONDS); + if (expiryTime > certificateGracePeriod) { + throw new IllegalArgumentException("Certificate grace period " + + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + + " should be greater than maximum block/container token lifetime " + + HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME); + } // Means this is an upgraded cluster and it has no sub-ca, // so SCM Certificate client is not initialized. To make Tokens @@ -926,10 +942,8 @@ private ContainerTokenSecretManager createContainerTokenSecretManager( scmCertificateClient = new SCMCertificateClient(securityConfig, certSerialNumber, SCM_ROOT_CA_COMPONENT_NAME); } - String certId = scmCertificateClient.getCertificate().getSerialNumber() - .toString(); return new ContainerTokenSecretManager(securityConfig, - expiryTime, certId); + expiryTime); } /** diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java index 2a8a945a0b27..7501d2855dd1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java @@ -681,10 +681,10 @@ public static void prepareData(int[][] ranges) throws Exception { SecurityConfig conf = new SecurityConfig(tweakedConfig); long tokenLifetime = TimeUnit.DAYS.toMillis(1); containerTokenGenerator = new ContainerTokenSecretManager( - conf, tokenLifetime, "1"); + conf, tokenLifetime); containerTokenGenerator.start(certClient); blockTokenGenerator = new OzoneBlockTokenSecretManager( - conf, tokenLifetime, "1"); + conf, tokenLifetime); blockTokenGenerator.start(certClient); containerToken = containerTokenGenerator .generateToken(ANY_USER, new ContainerID(containerID)); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index 95f07d7d3102..f89e04331c0c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.ozone; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; import java.io.File; import java.io.IOException; import java.net.InetAddress; @@ -31,9 +33,7 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; import java.util.Date; -import java.util.List; import java.util.Properties; import java.util.UUID; import java.util.concurrent.Callable; @@ -49,22 +49,23 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.HddsTestUtils; +import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.ha.HASecurityUtils; import org.apache.hadoop.hdds.scm.ha.SCMHANodeDetails; import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; +import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.hdds.scm.proxy.SCMContainerLocationFailoverProxyProvider; import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.security.exception.SCMSecurityException; +import org.apache.hadoop.hdds.security.token.ContainerTokenIdentifier; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultApprover; import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile; import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient; -import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer; -import org.apache.hadoop.hdds.security.x509.certificate.client.DefaultCertificateClient; -import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate; @@ -81,6 +82,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.client.CertificateClientTestImpl; import org.apache.hadoop.ozone.common.Storage; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMStorage; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -104,7 +106,11 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION_DEFAULT; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY; @@ -199,6 +205,8 @@ public final class TestSecureOzoneCluster { private OzoneManagerProtocolClientSideTranslatorPB omClient; private KeyPair keyPair; private Path omMetaDirPath; + private int certGraceTime = 10 * 1000; // 10s + private int delegationTokenMaxTime = 9 * 1000; // 9s @Before public void init() { @@ -227,7 +235,10 @@ public void init() { conf.set(OZONE_METADATA_DIRS, omMetaDirPath.toString()); conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name()); - conf.set(HDDS_X509_RENEW_GRACE_DURATION, "PT5S"); // 5s + conf.set(HDDS_X509_RENEW_GRACE_DURATION, + Duration.ofMillis(certGraceTime).toString()); + conf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, + delegationTokenMaxTime); workDir = GenericTestUtils.getTestDir(getClass().getSimpleName()); clusterId = UUID.randomUUID().toString(); @@ -1048,6 +1059,7 @@ public void testCertificateRotationUnRecoverableFailure() throws Exception { omStorage.setOmCertSerialId(certId); omStorage.forceInitialize(); + // second cert as renew response X509CertificateHolder newCertHolder = generateX509CertHolder(conf, null, null, Duration.ofSeconds(certificateLifetime)); DNCertificateClient mockClient = mock(DNCertificateClient.class); @@ -1077,58 +1089,73 @@ public void testCertificateRotationUnRecoverableFailure() throws Exception { public void testDelegationTokenRenewCrossCertificateRenew() throws Exception { try { // Setup secure OM for start. - int certLifetime = 15 * 1000; // seconds + final int certLifetime = 40 * 1000; // 40s OzoneConfiguration newConf = new OzoneConfiguration(conf); - int tokenMaxLifetime = certLifetime * 2; - newConf.setLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, tokenMaxLifetime); newConf.set(HDDS_X509_DEFAULT_DURATION, Duration.ofMillis(certLifetime).toString()); + newConf.set(HDDS_X509_RENEW_GRACE_DURATION, + Duration.ofMillis(certLifetime - 15 * 1000).toString()); + newConf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, + certLifetime - 20 * 1000); setupOm(newConf); OzoneManager.setTestSecureOmFlag(true); CertificateClientTestImpl certClient = - new CertificateClientTestImpl(newConf); + new CertificateClientTestImpl(newConf, true); X509Certificate omCert = certClient.getCertificate(); - System.out.println("OmCert " + omCert.getSerialNumber().toString()); + String omCertId1 = omCert.getSerialNumber().toString(); // Start OM om.setCertClient(certClient); om.start(); - GenericTestUtils.waitFor(() -> om.isLeaderReady(), 500, 5000); + GenericTestUtils.waitFor(() -> om.isLeaderReady(), 100, 10000); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); // Get first OM client which will authenticate via Kerberos omClient = new OzoneManagerProtocolClientSideTranslatorPB( - OmTransportFactory.create(conf, ugi, null), + OmTransportFactory.create(newConf, ugi, null), RandomStringUtils.randomAscii(5)); // Since client is already connected get a delegation token - Token token = omClient.getDelegationToken( + Token token1 = omClient.getDelegationToken( new Text("om")); // Check if token is of right kind and renewer is running om instance - assertNotNull(token); - assertEquals("OzoneToken", token.getKind().toString()); - assertEquals(OmUtils.getOmRpcAddress(conf), - token.getService().toString()); + assertNotNull(token1); + assertEquals("OzoneToken", token1.getKind().toString()); + assertEquals(OmUtils.getOmRpcAddress(newConf), + token1.getService().toString()); + OzoneTokenIdentifier temp = new OzoneTokenIdentifier(); + ByteArrayInputStream buf = new ByteArrayInputStream( + token1.getIdentifier()); + DataInputStream in = new DataInputStream(buf); + temp.readFields(in); + assertEquals(omCertId1, temp.getOmCertSerialId()); // Renew delegation token - long expiryTime = omClient.renewDelegationToken(token); + long expiryTime = omClient.renewDelegationToken(token1); assertTrue(expiryTime > 0); - // Wait for OM certificate to expire and renew - GenericTestUtils.waitFor(() -> omCert.getNotAfter().before(new Date()), - 500, certLifetime); - certClient.renewKey(); - assertFalse(omCert.getSerialNumber().toString().equals( - certClient.getCertificate().getSerialNumber().toString())); - System.out.println("New OmCert " + - certClient.getCertificate().getSerialNumber().toString()); - - // Test delegation renewal after certificate renewed; - expiryTime = omClient.renewDelegationToken(token); + // Wait for OM certificate to renew + GenericTestUtils.waitFor(() -> !omCertId1.equals( + certClient.getCertificate().getSerialNumber().toString()), + 100, certLifetime); + String omCertId2 = + certClient.getCertificate().getSerialNumber().toString(); + assertNotEquals(omCertId1, omCertId2); + // Get a new delegation token + Token token2 = omClient.getDelegationToken( + new Text("om")); + buf = new ByteArrayInputStream(token2.getIdentifier()); + in = new DataInputStream(buf); + temp.readFields(in); + assertEquals(omCertId2, temp.getOmCertSerialId()); + + // Because old certificate is still valid, so renew old token will succeed + expiryTime = omClient.renewDelegationToken(token1); assertTrue(expiryTime > 0); + assertTrue(new Date(expiryTime).before(omCert.getNotAfter())); } finally { if (om != null) { om.stop(); @@ -1138,128 +1165,159 @@ public void testDelegationTokenRenewCrossCertificateRenew() throws Exception { } /** - * Test functionality to get SCM signed certificate for OM. + * Tests container token renewal after a certificate renew */ @Test - public void testOMGrpcServerCertificateRenew() throws Exception { + public void testContainerTokenRenewCrossCertificateRenew() throws Exception { + // Setup secure SCM for start. + final int certLifetime = 40 * 1000; // 40s + conf.set(HDDS_X509_DEFAULT_DURATION, + Duration.ofMillis(certLifetime).toString()); + conf.set(HDDS_X509_MAX_DURATION, + Duration.ofMillis(certLifetime).toString()); + conf.set(HDDS_X509_RENEW_GRACE_DURATION, + Duration.ofMillis(certLifetime - 15 * 1000).toString()); + conf.setBoolean(HDDS_CONTAINER_TOKEN_ENABLED, true); + conf.setLong(HDDS_BLOCK_TOKEN_EXPIRY_TIME, certLifetime - 20 * 1000); + initSCM(); + scm = HddsTestUtils.getScmSimple(conf); try { - scm = HddsTestUtils.getScmSimple(conf); + CertificateClientTestImpl certClient = + new CertificateClientTestImpl(conf, true); + X509Certificate scmCert = certClient.getCertificate(); + String scmCertId1 = scmCert.getSerialNumber().toString(); + // Start SCM + scm.setScmCertificateClient(certClient); scm.start(); - int certLifetime = 30; // second - om = createOM(certLifetime, conf); - - X509Certificate omCert = om.getCertificateClient().getCertificate(); - X509Certificate caCert = om.getCertificateClient().getCACertificate(); - X509Certificate rootCaCert = - om.getCertificateClient().getRootCACertificate(); - List certList = new ArrayList<>(); - certList.add(caCert); - certList.add(rootCaCert); - // set certificates in GrpcOmTransport - GrpcOmTransport.setCaCerts(certList); - - GenericTestUtils.waitFor(() -> om.isLeaderReady(), 500, 10000); - String transportCls = GrpcOmTransportFactory.class.getName(); - conf.set(OZONE_OM_TRANSPORT_CLASS, transportCls); - OzoneClient client = OzoneClientFactory.getRpcClient(conf); - - ServiceInfoEx serviceInfoEx = client.getObjectStore() - .getClientProxy().getOzoneManagerClient().getServiceInfo(); - Assert.assertTrue(serviceInfoEx.getCaCertificate().equals( - CertificateCodec.getPEMEncodedString(caCert))); - - // Wait for OM certificate to expire and renew - GenericTestUtils.waitFor(() -> omCert.getNotAfter().before(new Date()), - 500, certLifetime * 1000); - - // rerun the command using old client, it should succeed - serviceInfoEx = client.getObjectStore() - .getClientProxy().getOzoneManagerClient().getServiceInfo(); - Assert.assertTrue(serviceInfoEx.getCaCertificate().equals( - CertificateCodec.getPEMEncodedString(caCert))); - client.close(); - - // rerun the command using new client, it should succeed too. - try { - OzoneClientFactory.getRpcClient(conf); - } catch (Exception e) { - fail("Create client should succeed for certificate is renewed"); - } + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + // Get SCM client which will authenticate via Kerberos + SCMContainerLocationFailoverProxyProvider proxyProvider = + new SCMContainerLocationFailoverProxyProvider(conf, ugi); + StorageContainerLocationProtocolClientSideTranslatorPB scmClient = + new StorageContainerLocationProtocolClientSideTranslatorPB( + proxyProvider); + + // Since client is already connected get a delegation token + ContainerID containerID = new ContainerID(1); + Token token1 = scmClient.getContainerToken(containerID); + + // Check if token is of right kind and renewer is running instance + assertNotNull(token1); + assertEquals(ContainerTokenIdentifier.KIND, token1.getKind()); + assertEquals(containerID.toString(), token1.getService().toString()); + ContainerTokenIdentifier temp = new ContainerTokenIdentifier(); + ByteArrayInputStream buf = new ByteArrayInputStream( + token1.getIdentifier()); + DataInputStream in = new DataInputStream(buf); + temp.readFields(in); + assertEquals(scmCertId1, temp.getCertSerialId()); + + // Wait for SCM certificate to renew + GenericTestUtils.waitFor(() -> !scmCertId1.equals( + certClient.getCertificate().getSerialNumber().toString()), + 100, certLifetime); + String scmCertId2 = + certClient.getCertificate().getSerialNumber().toString(); + assertNotEquals(scmCertId1, scmCertId2); + + // Get a new container token + containerID = new ContainerID(2); + Token token2 = scmClient.getContainerToken(containerID); + buf = new ByteArrayInputStream(token2.getIdentifier()); + in = new DataInputStream(buf); + temp.readFields(in); + assertEquals(scmCertId2, temp.getCertSerialId()); } finally { - DefaultCertificateClient.setUgi(null); - OzoneManager.setUgi(null); - GrpcOmTransport.setCaCerts(null); if (scm != null) { scm.stop(); } - if (om != null) { - om.stop(); - } - IOUtils.closeQuietly(om); } } /** - * Create an OzoneManager instance. - * @param certLifetime lifetime in seconds. - * @return - * @throws Exception + * Tests block token renewal after a certificate renew */ - private OzoneManager createOM(int certLifetime, OzoneConfiguration config) - throws Exception { - config.set(OZONE_METADATA_DIRS, omMetaDirPath.toString()); - config.set(HDDS_X509_DEFAULT_DURATION, - Duration.ofSeconds(certLifetime).toString()); - config.set(HDDS_SECURITY_SSL_KEYSTORE_RELOAD_INTERVAL, "1s"); - config.set(HDDS_SECURITY_SSL_TRUSTSTORE_RELOAD_INTERVAL, "1s"); + @Test + public void testBlockTokenRenewCrossCertificateRenew() throws Exception { + try { + // Setup secure OM for start. + final int certLifetime = 40 * 1000; // 40s + OzoneConfiguration newConf = new OzoneConfiguration(conf); + newConf.set(HDDS_X509_DEFAULT_DURATION, + Duration.ofMillis(certLifetime).toString()); + newConf.set(HDDS_X509_RENEW_GRACE_DURATION, + Duration.ofMillis(certLifetime - 15 * 1000).toString()); + newConf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, + certLifetime - 20 * 1000); - // initialize OmStorage, save om Cert and CA Certs to disk - OMStorage omStore = new OMStorage(config); - omStore.setClusterId(clusterId); - omStore.setOmId(omId); - SecurityConfig securityConfig = new SecurityConfig(config); - SCMCertificateClient scmCertClient = scm.getScmCertificateClient(); - CertificateCodec certCodec = new CertificateCodec(securityConfig, "om"); - X509Certificate scmCert = scmCertClient.getCertificate(); - X509Certificate rootCert = scmCertClient.getCACertificate(); - X509CertificateHolder certHolder = generateX509CertHolder(config, keyPair, - new KeyPair(scmCertClient.getPublicKey(), - scmCertClient.getPrivateKey()), scmCert, - Duration.ofSeconds(certLifetime)); - String certId = certHolder.getSerialNumber().toString(); - certCodec.writeCertificate(certHolder); - certCodec.writeCertificate(CertificateCodec.getCertificateHolder(scmCert), - String.format(DefaultCertificateClient.CERT_FILE_NAME_FORMAT, - DefaultCertificateClient.CA_CERT_PREFIX + - scmCert.getSerialNumber().toString()), false); - certCodec.writeCertificate(CertificateCodec.getCertificateHolder( - scmCertClient.getCACertificate()), - String.format(DefaultCertificateClient.CERT_FILE_NAME_FORMAT, - DefaultCertificateClient.ROOT_CA_CERT_PREFIX + - rootCert.getSerialNumber().toString()), false); - omStore.setOmCertSerialId(certId); - omStore.initialize(); + setupOm(newConf); + OzoneManager.setTestSecureOmFlag(true); - config.setBoolean(HDDS_GRPC_TLS_ENABLED, true); - config.setBoolean(OZONE_OM_S3_GPRC_SERVER_ENABLED, true); - OzoneManager.setTestSecureOmFlag(true); - UserGroupInformation ugi = - UserGroupInformation.loginUserFromKeytabAndReturnUGI( - config.get(OZONE_OM_KERBEROS_PRINCIPAL_KEY), - omKeyTab.getCanonicalPath()); - // In this process, SCM has already login using Kerberos. So pass - // specific UGI to DefaultCertificateClient and OzoneManager to avoid - // conflict with SCM procedure. - DefaultCertificateClient.setUgi(ugi); - OzoneManager.setUgi(ugi); - OzoneManager ozoneManager = OzoneManager.createOm(config); - ozoneManager.start(); - return ozoneManager; + CertificateClientTestImpl certClient = + new CertificateClientTestImpl(newConf, true); + X509Certificate omCert = certClient.getCertificate(); + String omCertId1 = omCert.getSerialNumber().toString(); + // Start OM + om.setCertClient(certClient); + om.start(); + GenericTestUtils.waitFor(() -> om.isLeaderReady(), 100, 10000); + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + + // Get first OM client which will authenticate via Kerberos + omClient = new OzoneManagerProtocolClientSideTranslatorPB( + OmTransportFactory.create(newConf, ugi, null), + RandomStringUtils.randomAscii(5)); + + // Since client is already connected get a delegation token + Token token1 = omClient.getDelegationToken( + new Text("om")); + + // Check if token is of right kind and renewer is running om instance + assertNotNull(token1); + assertEquals("OzoneToken", token1.getKind().toString()); + assertEquals(OmUtils.getOmRpcAddress(newConf), + token1.getService().toString()); + OzoneTokenIdentifier temp = new OzoneTokenIdentifier(); + ByteArrayInputStream buf = new ByteArrayInputStream( + token1.getIdentifier()); + DataInputStream in = new DataInputStream(buf); + temp.readFields(in); + assertEquals(omCertId1, temp.getOmCertSerialId()); + + // Renew delegation token + long expiryTime = omClient.renewDelegationToken(token1); + assertTrue(expiryTime > 0); + + // Wait for OM certificate to renew + GenericTestUtils.waitFor(() -> !omCertId1.equals( + certClient.getCertificate().getSerialNumber().toString()), + 100, certLifetime); + String omCertId2 = + certClient.getCertificate().getSerialNumber().toString(); + assertNotEquals(omCertId1, omCertId2); + // Get a new delegation token + Token token2 = omClient.getDelegationToken( + new Text("om")); + buf = new ByteArrayInputStream(token2.getIdentifier()); + in = new DataInputStream(buf); + temp.readFields(in); + assertEquals(omCertId2, temp.getOmCertSerialId()); + + // Because old certificate is still valid, so renew old token will succeed + expiryTime = omClient.renewDelegationToken(token1); + assertTrue(expiryTime > 0); + assertTrue(new Date(expiryTime).before(omCert.getNotAfter())); + } finally { + if (om != null) { + om.stop(); + } + IOUtils.closeQuietly(om); + } } ->>>>>>> 624df3bf0 (HDDS-7723. Refresh Keys and Certificate used in OzoneSecretManager after certificate renewed) public void validateCertificate(X509Certificate cert) throws Exception { // Assert that we indeed have a self signed certificate. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java index 77dc76eecb87..12b7523a4d41 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java @@ -34,16 +34,23 @@ import java.time.ZoneId; import java.util.Collections; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultApprover; import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; +import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateNotification; import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate; import org.apache.hadoop.hdds.security.x509.crl.CRLInfo; @@ -79,12 +86,15 @@ public class CertificateClientTestImpl implements CertificateClient { private KeyStoresFactory serverKeyStoresFactory; private KeyStoresFactory clientKeyStoresFactory; private Map certificateMap; + private ScheduledExecutorService executorService; + private Set notificationReceivers; - public CertificateClientTestImpl(OzoneConfiguration conf) throws Exception { - this(conf, true); + public CertificateClientTestImpl(OzoneConfiguration conf) + throws Exception { + this(conf, false); } - public CertificateClientTestImpl(OzoneConfiguration conf, boolean rootCA) + public CertificateClientTestImpl(OzoneConfiguration conf, boolean autoRenew) throws Exception { certificateMap = new ConcurrentHashMap<>(); securityConfig = new SecurityConfig(conf); @@ -144,6 +154,23 @@ public CertificateClientTestImpl(OzoneConfiguration conf, boolean rootCA) securityConfig, this, true); clientKeyStoresFactory = SecurityUtil.getClientKeyStoresFactory( securityConfig, this, true); + + if (autoRenew) { + Duration gracePeriod = securityConfig.getRenewalGracePeriod(); + Date expireDate = x509Certificate.getNotAfter(); + LocalDateTime gracePeriodStart = expireDate.toInstant() + .atZone(ZoneId.systemDefault()).toLocalDateTime().minus(gracePeriod); + LocalDateTime currentTime = LocalDateTime.now(); + Duration delay = gracePeriodStart.isBefore(currentTime) ? Duration.ZERO : + Duration.between(currentTime, gracePeriodStart); + + executorService = Executors.newScheduledThreadPool(1, + new ThreadFactoryBuilder().setNameFormat("CertificateLifetimeMonitor") + .setDaemon(true).build()); + this.executorService.schedule(new RenewCertTask(), + delay.toMillis(), TimeUnit.MILLISECONDS); + } + notificationReceivers = new HashSet<>(); } @Override @@ -368,11 +395,28 @@ public void renewKey() throws Exception { // Save the new private key and certificate to file // Save certificate and private key to keyStore + X509Certificate oldCert = x509Certificate; keyPair = newKeyPair; x509Certificate = newX509Certificate; certificateMap.put(x509Certificate.getSerialNumber().toString(), x509Certificate); System.out.println(new Date() + " certificated is renewed"); + + // notify notification receivers + notificationReceivers.forEach(r -> r.notifyCertificateRenewed( + oldCert.getSerialNumber().toString(), + x509Certificate.getSerialNumber().toString())); + } + + public class RenewCertTask implements Runnable{ + @Override + public void run() { + try { + renewKey(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } @Override @@ -385,6 +429,13 @@ public KeyStoresFactory getClientKeyStoresFactory() { return clientKeyStoresFactory; } + @Override + public void registerNotificationReceiver(CertificateNotification receiver) { + synchronized (notificationReceivers) { + notificationReceivers.add(receiver); + } + } + @Override public void close() throws IOException { if (serverKeyStoresFactory != null) { @@ -394,5 +445,9 @@ public void close() throws IOException { if (clientKeyStoresFactory != null) { clientKeyStoresFactory.destroy(); } + + if (executorService != null) { + executorService.shutdown(); + } } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java index 82af642e5b5a..cbf57a96d8d0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java @@ -119,8 +119,7 @@ public static void init() throws Exception { .setCertificateClient(certificateClientTest) .build(); secretManager = new OzoneBlockTokenSecretManager(new SecurityConfig(conf), - 60 * 60, certificateClientTest.getCertificate(). - getSerialNumber().toString()); + 60 * 60); secretManager.start(certificateClientTest); cluster.getOzoneManager().startSecretManager(); cluster.waitForClusterToBeReady(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java index 19fe829efe1a..ad96d4fa4b5f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java @@ -146,9 +146,9 @@ public void setup() throws Exception { HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, "1s", TimeUnit.MILLISECONDS); - caClient = new CertificateClientTestImpl(conf, false); + caClient = new CertificateClientTestImpl(conf); secretManager = new ContainerTokenSecretManager(new SecurityConfig(conf), - expiryTime, caClient.getCertificate().getSerialNumber().toString()); + expiryTime); } @Test(expected = CertificateExpiredException.class) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java index c94bfb20fd00..17dc2b34570b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java @@ -55,12 +55,14 @@ import java.io.IOException; import java.security.PrivilegedAction; +import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.UUID; -import java.util.concurrent.TimeUnit; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT; @@ -94,6 +96,9 @@ public class TestSecureOzoneContainer { private final boolean tokenExpired; private CertificateClientTestImpl caClient; private ContainerTokenSecretManager secretManager; + private int certMaxTime = 15 * 1000; // 15s + private int certGraceTime = 10 * 1000; // 10s + private int tokenMaxTime = 9 * 1000; // 9s public TestSecureOzoneContainer(Boolean requireToken, Boolean hasToken, Boolean tokenExpired) { @@ -120,12 +125,15 @@ public void setup() throws Exception { String ozoneMetaPath = GenericTestUtils.getTempPath("ozoneMeta"); conf.set(OZONE_METADATA_DIRS, ozoneMetaPath); + conf.set(HDDS_X509_DEFAULT_DURATION, Duration.ofMillis(certMaxTime).toString()); + conf.set(HDDS_X509_RENEW_GRACE_DURATION, + Duration.ofMillis(certGraceTime).toString()); + conf.setLong(HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, + tokenMaxTime); secConfig = new SecurityConfig(conf); - caClient = new CertificateClientTestImpl(conf); + caClient = new CertificateClientTestImpl(conf, true); secretManager = new ContainerTokenSecretManager( - new SecurityConfig(conf), - TimeUnit.DAYS.toMillis(1), - caClient.getCertificate().getSerialNumber().toString()); + new SecurityConfig(conf), tokenMaxTime); } @Test @@ -206,6 +214,80 @@ public void testCreateOzoneContainer() throws Exception { } } + @Test + public void testContainerTokenWithCertRenew() throws Exception { + conf.setBoolean(HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED, true); + + ContainerID containerID = ContainerID.valueOf(getTestContainerID()); + OzoneContainer container = null; + System.out.println(System.getProperties().getProperty("java.library.path")); + try { + Pipeline pipeline = MockPipeline.createSingleNodePipeline(); + conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.getRoot().getPath()); + conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline + .getFirstNode().getPort(DatanodeDetails.Port.Name.STANDALONE) + .getValue()); + conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false); + + DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); + container = new OzoneContainer(dn, conf, getContext(dn), caClient); + //Set scmId and manually start ozone container. + container.start(UUID.randomUUID().toString()); + + String user = "user1"; + UserGroupInformation ugi = UserGroupInformation.createUserForTesting( + user, new String[] {"usergroup"}); + + int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue(); + if (port == 0) { + port = secConfig.getConfiguration().getInt(OzoneConfigKeys + .DFS_CONTAINER_IPC_PORT, DFS_CONTAINER_IPC_PORT_DEFAULT); + } + secretManager.start(caClient); + + ugi.doAs((PrivilegedAction) () -> { + try { + XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf); + client.connect(); + + Token token = null; + if (hasToken) { + Instant expiryDate = tokenExpired + ? Instant.now().minusSeconds(3600) + : Instant.now().plusSeconds(3600); + ContainerTokenIdentifier tokenIdentifier = + new ContainerTokenIdentifier(user, containerID, + caClient.getCertificate().getSerialNumber().toString(), + expiryDate); + token = secretManager.generateToken(tokenIdentifier); + } + + ContainerCommandRequestProto request = + getCreateContainerSecureRequest(containerID.getId(), + client.getPipeline(), token); + ContainerCommandResponseProto response = client.sendCommand(request); + assertNotNull(response); + ContainerProtos.Result expectedResult = + !requireToken || (hasToken && !tokenExpired) + ? ContainerProtos.Result.SUCCESS + : ContainerProtos.Result.BLOCK_TOKEN_VERIFICATION_FAILED; + assertEquals(expectedResult, response.getResult(), this::testCase); + } catch (SCMSecurityException e) { + assertState(requireToken && hasToken && tokenExpired); + } catch (IOException e) { + assertState(requireToken && !hasToken); + } catch (Exception e) { + fail(e); + } + return null; + }); + } finally { + if (container != null) { + container.stop(); + } + } + } + private StateContext getContext(DatanodeDetails datanodeDetails) { DatanodeStateMachine stateMachine = Mockito.mock( DatanodeStateMachine.class); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java index 62e7ecbda682..4b87ed1e7a3c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java @@ -130,17 +130,15 @@ public static void setup() throws Exception { CONF.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true); caClient = new CertificateClientTestImpl(CONF); - String certSerialId = - caClient.getCertificate().getSerialNumber().toString(); SecurityConfig secConf = new SecurityConfig(CONF); long tokenLifetime = TimeUnit.HOURS.toMillis(1); blockTokenSecretManager = new OzoneBlockTokenSecretManager( - secConf, tokenLifetime, certSerialId); + secConf, tokenLifetime); blockTokenSecretManager.start(caClient); containerTokenSecretManager = new ContainerTokenSecretManager( - secConf, tokenLifetime, certSerialId); + secConf, tokenLifetime); containerTokenSecretManager.start(caClient); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 1f00534f65a3..a4015ae60c6c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -955,9 +955,11 @@ private OzoneDelegationTokenSecretManager createDelegationTokenSecretManager( conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); - Duration certificateGracePeriod = - new SecurityConfig(conf).getRenewalGracePeriod(); - if (certificateGracePeriod.toMillis() < tokenMaxLifetime) { + long certificateGracePeriod = + conf.getTimeDuration(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT, + TimeUnit.MILLISECONDS); + if (tokenMaxLifetime > certificateGracePeriod) { throw new IllegalArgumentException("Certificate grace period " + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + " should be greater than maximum delegation token lifetime " + @@ -984,13 +986,22 @@ private OzoneBlockTokenSecretManager createBlockTokenSecretManager( HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT, TimeUnit.MILLISECONDS); + long certificateGracePeriod = + conf.getTimeDuration(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT, + TimeUnit.MILLISECONDS); + if (expiryTime > certificateGracePeriod) { + throw new IllegalArgumentException("Certificate grace period " + + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + + " should be greater than maximum block token lifetime " + + HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME); + } // TODO: Pass OM cert serial ID. if (testSecureOmFlag) { - return new OzoneBlockTokenSecretManager(secConfig, expiryTime, "1"); + return new OzoneBlockTokenSecretManager(secConfig, expiryTime); } Objects.requireNonNull(certClient); - return new OzoneBlockTokenSecretManager(secConfig, expiryTime, - certClient.getCertificate().getSerialNumber().toString()); + return new OzoneBlockTokenSecretManager(secConfig, expiryTime); } private void stopSecretManager() { @@ -2101,6 +2112,9 @@ public void stop() { OMPerformanceMetrics.unregister(); RatisDropwizardExports.clear(ratisMetricsMap, ratisReporterList); scmClient.close(); + if (certClient != null) { + certClient.close(); + } } catch (Exception e) { LOG.error("OzoneManager stop failed.", e); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java index 742726356f6f..513374da8275 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java @@ -264,17 +264,10 @@ private void updateIdentifierDetails(OzoneTokenIdentifier identifier) { identifier.setMasterKeyId(getCurrentKey().getKeyId()); identifier.setSequenceNumber(sequenceNum); identifier.setMaxDate(now + getTokenMaxLifetime()); - identifier.setOmCertSerialId(getOmCertificateSerialId()); + identifier.setOmCertSerialId(getCertSerialId()); identifier.setOmServiceId(getOmServiceId()); } - /** - * Get OM certificate serial id. - * */ - private String getOmCertificateSerialId() { - return getCertClient().getCertificate().getSerialNumber().toString(); - } - private String getOmServiceId() { return omServiceId; } From 1f51be2d33793f0bd7b11c9cf8e85c65db058f1a Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Mon, 16 Jan 2023 21:04:15 +0800 Subject: [PATCH 3/7] refinement --- .../apache/hadoop/hdds/HddsConfigKeys.java | 3 - .../client/DefaultCertificateClient.java | 15 +--- .../hadoop/hdds/utils/HddsServerUtil.java | 6 +- .../ozoneimpl/TestSecureOzoneContainer.java | 90 +------------------ .../apache/hadoop/ozone/om/OzoneManager.java | 13 +-- .../OzoneDelegationTokenSecretManager.java | 6 +- 6 files changed, 14 insertions(+), 119 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java index 705fd93144e0..cb258dfa74dc 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java @@ -195,9 +195,6 @@ public final class HddsConfigKeys { public static final String HDDS_X509_RENEW_GRACE_DURATION_DEFAULT = "P28D"; public static final String HDDS_NEW_KEY_CERT_DIR_NAME_SUFFIX = "-next"; public static final String HDDS_BACKUP_KEY_CERT_DIR_NAME_SUFFIX = "-previous"; - public static final String HDDS_X509_RENEW_SCAN_INTERVAL = - "hdds.x509.renew.scan.interval"; - public static final String HDDS_X509_RENEW_SCAN_INTERVAL_DEFAULT = "60s"; public static final String HDDS_CONTAINER_REPLICATION_COMPRESSION = "hdds.container.replication.compression"; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java index e5d719c7668f..21a7dc59d301 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java @@ -97,7 +97,6 @@ import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateException.ErrorCode.ROLLBACK_ERROR; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClientWithMaxRetry; -import org.apache.hadoop.security.UserGroupInformation; import org.bouncycastle.cert.X509CertificateHolder; import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; @@ -111,10 +110,10 @@ public abstract class DefaultCertificateClient implements CertificateClient { private static final Random RANDOM = new SecureRandom(); - public static final String CERT_FILE_NAME_FORMAT = "%s.crt"; - public static final String CA_CERT_PREFIX = "CA-"; + private static final String CERT_FILE_NAME_FORMAT = "%s.crt"; + private static final String CA_CERT_PREFIX = "CA-"; private static final int CA_CERT_PREFIX_LEN = 3; - public static final String ROOT_CA_CERT_PREFIX = "ROOTCA-"; + private static final String ROOT_CA_CERT_PREFIX = "ROOTCA-"; private static final int ROOT_CA_PREFIX_LEN = 7; private final Logger logger; private final SecurityConfig securityConfig; @@ -146,7 +145,6 @@ public abstract class DefaultCertificateClient implements CertificateClient { private Runnable shutdownCallback; private SCMSecurityProtocolClientSideTranslatorPB scmSecurityProtocolClient; private Set notificationReceivers; - private static UserGroupInformation ugi; DefaultCertificateClient(SecurityConfig securityConfig, Logger log, String certSerialId, String component, @@ -1378,7 +1376,7 @@ public SCMSecurityProtocolClientSideTranslatorPB getScmSecureClient() if (scmSecurityProtocolClient == null) { scmSecurityProtocolClient = getScmSecurityClientWithMaxRetry( - (OzoneConfiguration) securityConfig.getConfiguration(), ugi); + (OzoneConfiguration) securityConfig.getConfiguration()); } return scmSecurityProtocolClient; } @@ -1389,11 +1387,6 @@ public void setSecureScmClient( scmSecurityProtocolClient = client; } - @VisibleForTesting - public static void setUgi(UserGroupInformation user) { - ugi = user; - } - public synchronized void startCertificateMonitor() { Preconditions.checkNotNull(getCertificate(), "Component certificate should not be empty"); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java index af3c5d5ed475..80970ff1bf65 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java @@ -450,8 +450,8 @@ public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient( } public static SCMSecurityProtocolClientSideTranslatorPB - getScmSecurityClientWithMaxRetry(OzoneConfiguration conf, - UserGroupInformation ugi) throws IOException { + getScmSecurityClientWithMaxRetry(OzoneConfiguration conf) + throws IOException { // Certificate from SCM is required for DN startup to succeed, so retry // for ever. In this way DN start up is resilient to SCM service running // status. @@ -464,7 +464,7 @@ public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient( return new SCMSecurityProtocolClientSideTranslatorPB( new SCMSecurityProtocolFailoverProxyProvider(configuration, - ugi == null ? UserGroupInformation.getCurrentUser() : ugi)); + UserGroupInformation.getCurrentUser())); } public static SCMSecurityProtocolClientSideTranslatorPB diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java index 17dc2b34570b..95eeadc08e9c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java @@ -55,14 +55,12 @@ import java.io.IOException; import java.security.PrivilegedAction; -import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.UUID; +import java.util.concurrent.TimeUnit; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT; @@ -96,9 +94,6 @@ public class TestSecureOzoneContainer { private final boolean tokenExpired; private CertificateClientTestImpl caClient; private ContainerTokenSecretManager secretManager; - private int certMaxTime = 15 * 1000; // 15s - private int certGraceTime = 10 * 1000; // 10s - private int tokenMaxTime = 9 * 1000; // 9s public TestSecureOzoneContainer(Boolean requireToken, Boolean hasToken, Boolean tokenExpired) { @@ -125,15 +120,10 @@ public void setup() throws Exception { String ozoneMetaPath = GenericTestUtils.getTempPath("ozoneMeta"); conf.set(OZONE_METADATA_DIRS, ozoneMetaPath); - conf.set(HDDS_X509_DEFAULT_DURATION, Duration.ofMillis(certMaxTime).toString()); - conf.set(HDDS_X509_RENEW_GRACE_DURATION, - Duration.ofMillis(certGraceTime).toString()); - conf.setLong(HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, - tokenMaxTime); secConfig = new SecurityConfig(conf); - caClient = new CertificateClientTestImpl(conf, true); + caClient = new CertificateClientTestImpl(conf); secretManager = new ContainerTokenSecretManager( - new SecurityConfig(conf), tokenMaxTime); + new SecurityConfig(conf), TimeUnit.DAYS.toMillis(1)); } @Test @@ -214,80 +204,6 @@ public void testCreateOzoneContainer() throws Exception { } } - @Test - public void testContainerTokenWithCertRenew() throws Exception { - conf.setBoolean(HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED, true); - - ContainerID containerID = ContainerID.valueOf(getTestContainerID()); - OzoneContainer container = null; - System.out.println(System.getProperties().getProperty("java.library.path")); - try { - Pipeline pipeline = MockPipeline.createSingleNodePipeline(); - conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.getRoot().getPath()); - conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline - .getFirstNode().getPort(DatanodeDetails.Port.Name.STANDALONE) - .getValue()); - conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false); - - DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails(); - container = new OzoneContainer(dn, conf, getContext(dn), caClient); - //Set scmId and manually start ozone container. - container.start(UUID.randomUUID().toString()); - - String user = "user1"; - UserGroupInformation ugi = UserGroupInformation.createUserForTesting( - user, new String[] {"usergroup"}); - - int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue(); - if (port == 0) { - port = secConfig.getConfiguration().getInt(OzoneConfigKeys - .DFS_CONTAINER_IPC_PORT, DFS_CONTAINER_IPC_PORT_DEFAULT); - } - secretManager.start(caClient); - - ugi.doAs((PrivilegedAction) () -> { - try { - XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf); - client.connect(); - - Token token = null; - if (hasToken) { - Instant expiryDate = tokenExpired - ? Instant.now().minusSeconds(3600) - : Instant.now().plusSeconds(3600); - ContainerTokenIdentifier tokenIdentifier = - new ContainerTokenIdentifier(user, containerID, - caClient.getCertificate().getSerialNumber().toString(), - expiryDate); - token = secretManager.generateToken(tokenIdentifier); - } - - ContainerCommandRequestProto request = - getCreateContainerSecureRequest(containerID.getId(), - client.getPipeline(), token); - ContainerCommandResponseProto response = client.sendCommand(request); - assertNotNull(response); - ContainerProtos.Result expectedResult = - !requireToken || (hasToken && !tokenExpired) - ? ContainerProtos.Result.SUCCESS - : ContainerProtos.Result.BLOCK_TOKEN_VERIFICATION_FAILED; - assertEquals(expectedResult, response.getResult(), this::testCase); - } catch (SCMSecurityException e) { - assertState(requireToken && hasToken && tokenExpired); - } catch (IOException e) { - assertState(requireToken && !hasToken); - } catch (Exception e) { - fail(e); - } - return null; - }); - } finally { - if (container != null) { - container.stop(); - } - } - } - private StateContext getContext(DatanodeDetails datanodeDetails) { DatanodeStateMachine stateMachine = Mockito.mock( DatanodeStateMachine.class); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index a4015ae60c6c..1045a9a98a82 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -412,7 +412,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl // Test flags private static boolean testReloadConfigFlag = false; private static boolean testSecureOmFlag = false; - private static UserGroupInformation ugi; private final OzoneLockProvider ozoneLockProvider; private OMPerformanceMetrics perfMetrics; @@ -1235,7 +1234,7 @@ private static boolean isOzoneSecurityEnabled() { private static void loginOMUserIfSecurityEnabled(OzoneConfiguration conf) throws IOException, AuthenticationException { securityEnabled = OzoneSecurityUtil.isSecurityEnabled(conf); - if (securityEnabled && ugi == null) { + if (securityEnabled) { loginOMUser(conf); } } @@ -3967,16 +3966,6 @@ public static void setTestSecureOmFlag(boolean testSecureOmFlag) { OzoneManager.testSecureOmFlag = testSecureOmFlag; } - @VisibleForTesting - public static void setUgi(UserGroupInformation user) { - OzoneManager.ugi = user; - } - - @VisibleForTesting - public static UserGroupInformation getUgi() { - return OzoneManager.ugi; - } - public OMNodeDetails getNodeDetails() { return omNodeDetails; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java index 513374da8275..0565c1507a92 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/OzoneDelegationTokenSecretManager.java @@ -214,9 +214,9 @@ public Token createToken(Text owner, Text renewer, Token token = new Token<>(identifier.getBytes(), password, identifier.getKind(), getService()); - //if (LOG.isDebugEnabled()) { - LOG.info("Created delegation token: {}", token); - //} + if (LOG.isDebugEnabled()) { + LOG.debug("Created delegation token: {}", token); + } return token; } From 89317310bec53c76405b91fe8c5e6135fe1e416c Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Mon, 16 Jan 2023 21:18:28 +0800 Subject: [PATCH 4/7] fix checkstyle --- .../scm/server/StorageContainerManager.java | 8 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 118 +----------------- .../client/CertificateClientTestImpl.java | 5 +- .../apache/hadoop/ozone/om/OzoneManager.java | 14 +-- 4 files changed, 16 insertions(+), 129 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 07a3e87fc473..0131d1b7a33b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -25,6 +25,7 @@ import com.google.common.base.Preconditions; import com.google.protobuf.BlockingService; +import java.time.Duration; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; @@ -909,10 +910,9 @@ private ContainerTokenSecretManager createContainerTokenSecretManager( HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT, TimeUnit.MILLISECONDS); - long certificateGracePeriod = - conf.getTimeDuration(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, - HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT, - TimeUnit.MILLISECONDS); + long certificateGracePeriod = Duration.parse( + conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT)).toMillis(); if (expiryTime > certificateGracePeriod) { throw new IllegalArgumentException("Certificate grace period " + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index f89e04331c0c..7693afdeb030 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -1083,7 +1083,7 @@ public void testCertificateRotationUnRecoverableFailure() throws Exception { } /** - * Tests delegation token renewal after a certificate renew + * Tests delegation token renewal after a certificate renew. */ @Test public void testDelegationTokenRenewCrossCertificateRenew() throws Exception { @@ -1165,7 +1165,7 @@ public void testDelegationTokenRenewCrossCertificateRenew() throws Exception { } /** - * Tests container token renewal after a certificate renew + * Tests container token renewal after a certificate renew. */ @Test public void testContainerTokenRenewCrossCertificateRenew() throws Exception { @@ -1236,88 +1236,6 @@ public void testContainerTokenRenewCrossCertificateRenew() throws Exception { } } - /** - * Tests block token renewal after a certificate renew - */ - @Test - public void testBlockTokenRenewCrossCertificateRenew() throws Exception { - try { - // Setup secure OM for start. - final int certLifetime = 40 * 1000; // 40s - OzoneConfiguration newConf = new OzoneConfiguration(conf); - newConf.set(HDDS_X509_DEFAULT_DURATION, - Duration.ofMillis(certLifetime).toString()); - newConf.set(HDDS_X509_RENEW_GRACE_DURATION, - Duration.ofMillis(certLifetime - 15 * 1000).toString()); - newConf.setLong(OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY, - certLifetime - 20 * 1000); - - setupOm(newConf); - OzoneManager.setTestSecureOmFlag(true); - - CertificateClientTestImpl certClient = - new CertificateClientTestImpl(newConf, true); - X509Certificate omCert = certClient.getCertificate(); - String omCertId1 = omCert.getSerialNumber().toString(); - // Start OM - om.setCertClient(certClient); - om.start(); - GenericTestUtils.waitFor(() -> om.isLeaderReady(), 100, 10000); - - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - - // Get first OM client which will authenticate via Kerberos - omClient = new OzoneManagerProtocolClientSideTranslatorPB( - OmTransportFactory.create(newConf, ugi, null), - RandomStringUtils.randomAscii(5)); - - // Since client is already connected get a delegation token - Token token1 = omClient.getDelegationToken( - new Text("om")); - - // Check if token is of right kind and renewer is running om instance - assertNotNull(token1); - assertEquals("OzoneToken", token1.getKind().toString()); - assertEquals(OmUtils.getOmRpcAddress(newConf), - token1.getService().toString()); - OzoneTokenIdentifier temp = new OzoneTokenIdentifier(); - ByteArrayInputStream buf = new ByteArrayInputStream( - token1.getIdentifier()); - DataInputStream in = new DataInputStream(buf); - temp.readFields(in); - assertEquals(omCertId1, temp.getOmCertSerialId()); - - // Renew delegation token - long expiryTime = omClient.renewDelegationToken(token1); - assertTrue(expiryTime > 0); - - // Wait for OM certificate to renew - GenericTestUtils.waitFor(() -> !omCertId1.equals( - certClient.getCertificate().getSerialNumber().toString()), - 100, certLifetime); - String omCertId2 = - certClient.getCertificate().getSerialNumber().toString(); - assertNotEquals(omCertId1, omCertId2); - // Get a new delegation token - Token token2 = omClient.getDelegationToken( - new Text("om")); - buf = new ByteArrayInputStream(token2.getIdentifier()); - in = new DataInputStream(buf); - temp.readFields(in); - assertEquals(omCertId2, temp.getOmCertSerialId()); - - // Because old certificate is still valid, so renew old token will succeed - expiryTime = omClient.renewDelegationToken(token1); - assertTrue(expiryTime > 0); - assertTrue(new Date(expiryTime).before(omCert.getNotAfter())); - } finally { - if (om != null) { - om.stop(); - } - IOUtils.closeQuietly(om); - } - } - public void validateCertificate(X509Certificate cert) throws Exception { // Assert that we indeed have a self signed certificate. @@ -1385,36 +1303,4 @@ private static X509CertificateHolder generateX509CertHolder( .setScmID("test") .build(); } - - private static X509CertificateHolder generateX509CertHolder ( - OzoneConfiguration conf, KeyPair keyPair, KeyPair rootKeyPair, - X509Certificate rootCert, Duration certLifetime) throws Exception { - // Generate normal certificate, signed by RootCA certificate - SecurityConfig secConfig = new SecurityConfig(conf); - DefaultApprover approver = new DefaultApprover(new DefaultProfile(), - secConfig); - - CertificateSignRequest.Builder csrBuilder = - new CertificateSignRequest.Builder(); - // Get host name. - csrBuilder.setKey(keyPair) - .setConfiguration(conf) - .setScmID("test") - .setClusterID("cluster") - .setSubject("localhost") - .setDigitalSignature(true) - .setDigitalEncryption(true); - - LocalDateTime start = LocalDateTime.now(); - String certDuration = conf.get(HDDS_X509_DEFAULT_DURATION, - HDDS_X509_DEFAULT_DURATION_DEFAULT); - X509CertificateHolder certificateHolder = - approver.sign(secConfig, rootKeyPair.getPrivate(), - new X509CertificateHolder(rootCert.getEncoded()), - Date.from(start.atZone(ZoneId.systemDefault()).toInstant()), - Date.from(start.plus(Duration.parse(certDuration)) - .atZone(ZoneId.systemDefault()).toInstant()), - csrBuilder.build(), "test", "cluster"); - return certificateHolder; - } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java index 12b7523a4d41..627a62907d60 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java @@ -408,7 +408,10 @@ public void renewKey() throws Exception { x509Certificate.getSerialNumber().toString())); } - public class RenewCertTask implements Runnable{ + /** + * Task to renew certificate. + */ + public class RenewCertTask implements Runnable { @Override public void run() { try { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 1045a9a98a82..fbcedcf0b44a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -954,10 +954,9 @@ private OzoneDelegationTokenSecretManager createDelegationTokenSecretManager( conf.getTimeDuration(OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, OMConfigKeys.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS); - long certificateGracePeriod = - conf.getTimeDuration(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, - HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT, - TimeUnit.MILLISECONDS); + long certificateGracePeriod = Duration.parse( + conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT)).toMillis(); if (tokenMaxLifetime > certificateGracePeriod) { throw new IllegalArgumentException("Certificate grace period " + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + @@ -985,10 +984,9 @@ private OzoneBlockTokenSecretManager createBlockTokenSecretManager( HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT, TimeUnit.MILLISECONDS); - long certificateGracePeriod = - conf.getTimeDuration(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, - HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT, - TimeUnit.MILLISECONDS); + long certificateGracePeriod = Duration.parse( + conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT)).toMillis(); if (expiryTime > certificateGracePeriod) { throw new IllegalArgumentException("Certificate grace period " + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + From 275d1ea692e3c53d14d65f2d34ad93b237e77f97 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Mon, 16 Jan 2023 21:30:29 +0800 Subject: [PATCH 5/7] checkstyle --- .../x509/certificate/client/CertificateNotification.java | 4 ++++ .../java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java | 5 ----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java index 3595cf21a981..948acaf2f724 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateNotification.java @@ -19,6 +19,10 @@ package org.apache.hadoop.hdds.security.x509.certificate.client; +/** + * Class should implement this interface if it wants to be notified when there + * is some changes in Certificate. + */ public interface CertificateNotification { /** * Notify the class implementing this interface that certificate is renewed. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index 7693afdeb030..a55faa679d07 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -31,7 +31,6 @@ import java.time.Duration; import java.time.LocalDate; import java.time.LocalDateTime; -import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.Properties; @@ -63,11 +62,8 @@ import org.apache.hadoop.hdds.security.exception.SCMSecurityException; import org.apache.hadoop.hdds.security.token.ContainerTokenIdentifier; import org.apache.hadoop.hdds.security.x509.SecurityConfig; -import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultApprover; -import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile; import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient; import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; -import org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest; import org.apache.hadoop.hdds.security.x509.certificates.utils.SelfSignedCertificate; import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException; import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator; @@ -109,7 +105,6 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_TOKEN_ENABLED; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_DEFAULT_DURATION_DEFAULT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_MAX_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; From 46e4c38862f5e4f59aa06063ebd8f6a05cdd4d4d Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Mon, 16 Jan 2023 22:16:04 +0800 Subject: [PATCH 6/7] fix findbugs --- .../ozone/container/ec/reconstruction/TokenHelper.java | 9 +++++---- .../apache/hadoop/hdds/security/OzoneSecretManager.java | 6 ++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java index 53fee24bed8b..ffd763280476 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/TokenHelper.java @@ -32,6 +32,7 @@ import org.apache.hadoop.security.token.Token; import java.io.IOException; +import java.time.Duration; import java.util.EnumSet; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -69,10 +70,10 @@ class TokenHelper { HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME, HddsConfigKeys.HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT, TimeUnit.MILLISECONDS); - long certificateGracePeriod = - conf.getTimeDuration(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, - HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT, - TimeUnit.MILLISECONDS); + long certificateGracePeriod = Duration.parse( + conf.get(HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION, + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION_DEFAULT)) + .toMillis(); if (expiryTime > certificateGracePeriod) { throw new IllegalArgumentException("Certificate grace period " + HddsConfigKeys.HDDS_X509_RENEW_GRACE_DURATION + diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java index 6c8530684375..121d06a268f2 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java @@ -187,9 +187,11 @@ public void notifyCertificateRenewed(String oldCertId, String newCertId) { logger.info("Old certificate Id doesn't match. Holding {}, oldCertId {}", getCertSerialId(), oldCertId); } - if (!newCertId.equals(certClient.getCertificate())) { + if (!newCertId.equals( + certClient.getCertificate().getSerialNumber().toString())) { logger.info("New certificate Id doesn't match. Holding in caClient {}," + - " newCertId {}", newCertId, certClient.getCertificate()); + " newCertId {}", newCertId, + certClient.getCertificate().getSerialNumber().toString()); } updateCurrentKey(new KeyPair(certClient.getPublicKey(), certClient.getPrivateKey()), certClient.getCertificate()); From 0b7d3765a1da6d01dee0fa31daa4bf81414e4ba8 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Tue, 17 Jan 2023 21:16:34 +0800 Subject: [PATCH 7/7] fix failed TestContainerCommandsEC test --- .../org/apache/hadoop/hdds/security/OzoneSecretManager.java | 4 +++- .../hadoop/hdds/scm/storage/TestContainerCommandsEC.java | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java index 121d06a268f2..4b1bdc053d36 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/OzoneSecretManager.java @@ -172,7 +172,8 @@ public int incrementDelegationTokenSeqNum() { */ private OzoneSecretKey updateCurrentKey(KeyPair keyPair, X509Certificate certificate) { - logger.info("Updating the current master key for generating tokens"); + logger.info("Updating current master key for generating tokens. Cert id {}", + certificate.getSerialNumber().toString()); int newCurrentId = incrementCurrentKeyId(); OzoneSecretKey newKey = new OzoneSecretKey(newCurrentId, @@ -193,6 +194,7 @@ public void notifyCertificateRenewed(String oldCertId, String newCertId) { " newCertId {}", newCertId, certClient.getCertificate().getSerialNumber().toString()); } + logger.info("Certificate is changed from {} to {}", oldCertId, newCertId); updateCurrentKey(new KeyPair(certClient.getPublicKey(), certClient.getPrivateKey()), certClient.getCertificate()); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java index 7501d2855dd1..9b7b3d90c392 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java @@ -628,11 +628,11 @@ public static void startCluster(OzoneConfiguration conf) throws Exception { conf.setFromObject(writableECContainerProviderConfig); OzoneManager.setTestSecureOmFlag(true); - certClient = new CertificateClientTestImpl(config); + certClient = new CertificateClientTestImpl(conf); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(NUM_DN) .setScmId(SCM_ID).setClusterId(CLUSTER_ID) - .setCertificateClient(new CertificateClientTestImpl(conf)) + .setCertificateClient(certClient) .build(); cluster.waitForClusterToBeReady(); cluster.getOzoneManager().startSecretManager();