HDDS-4515. Datanodes should be able to persist and load CRL

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/utils/CRLCodec.java → hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLCodec.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  *
  */
-package org.apache.hadoop.hdds.security.x509.certificate.utils;
+package org.apache.hadoop.hdds.security.x509.crl;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.hdds.security.exception.SCMSecurityException;

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLInfo.java → hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLInfo.java

@@ -20,7 +20,6 @@
 
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.certificate.utils.CRLCodec;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/CRLInfoCodec.java → hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/crl/CRLInfoCodec.java

@@ -16,11 +16,10 @@
  * limitations under the License.
  *
  */
-package org.apache.hadoop.hdds.scm.metadata;
+package org.apache.hadoop.hdds.security.x509.crl;
 
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
 import org.apache.hadoop.hdds.utils.db.Codec;
 
 import java.io.IOException;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/crl/package-info.java

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+
+/**
+ * This package contains common routines used in creating an x509 CRL.
+ */
+package org.apache.hadoop.hdds.security.x509.crl;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/package-info.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains common db util classes.
+ */
+package org.apache.hadoop.hdds.utils.db;

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -125,7 +125,10 @@ public final class OzoneConsts {
    */
   public static final String CONTAINER_DB_SUFFIX = "container.db";
   public static final String PIPELINE_DB_SUFFIX = "pipeline.db";
+  public static final String CRL_DB_SUFFIX = "crl.db";
   public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
+  public static final String DN_CRL_DB = "dn-"+ CRL_DB_SUFFIX;
+  public static final String CRL_DB_DIRECTORY_NAME = "crl";
   public static final String OM_DB_NAME = "om.db";
   public static final String SCM_DB_NAME = "scm.db";
   public static final String OM_DB_BACKUP_PREFIX = "om.db.backup.";

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/CRLDBDefinition.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.datanode.metadata;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient;
+import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
+import org.apache.hadoop.hdds.security.x509.crl.CRLInfoCodec;
+import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
+import org.apache.hadoop.hdds.utils.db.DBDefinition;
+import org.apache.hadoop.hdds.utils.db.LongCodec;
+import org.apache.hadoop.hdds.utils.db.StringCodec;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+import java.io.File;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
+
+/**
+ * Class defines the structure and types of the crl.db.
+ */
+public class CRLDBDefinition implements DBDefinition {
+
+  public static final DBColumnFamilyDefinition<Long, CRLInfo> PENDING_CRLS =
+      new DBColumnFamilyDefinition<>(
+          "pendingCrls",
+          Long.class,
+          new LongCodec(),
+          CRLInfo.class,
+          new CRLInfoCodec());
+
+  public static final DBColumnFamilyDefinition<String, Long>
+      CRL_SEQUENCE_ID =
+      new DBColumnFamilyDefinition<>(
+          "crlSequenceId",
+          String.class,
+          new StringCodec(),
+          Long.class,
+          new LongCodec());
+
+  @Override
+  public String getName() {
+    return OzoneConsts.DN_CRL_DB;
+  }
+
+  @Override
+  public String getLocationConfigKey() {
+    throw new UnsupportedOperationException(
+        "No location config key available for datanode databases.");
+  }
+
+  @Override
+  public File getDBLocation(ConfigurationSource conf) {
+    // Please Note: To make it easy for our customers we will attempt to read
+    // HDDS metadata dir and if that is not set, we will use Ozone directory.
+    // TODO: We might want to fix this later.
+    String metadataDir = conf.get(HDDS_METADATA_DIR_NAME,
+        conf.get(OZONE_METADATA_DIRS,
+            conf.get(HDDS_DATANODE_DIR_KEY)));
+    Preconditions.checkNotNull(metadataDir, "Metadata directory can't be"
+        + " null. Please check configs.");
+
+    // create directories in the path if they do not already exist
+    HddsUtils.createDir(metadataDir
+        + File.separator
+        + DNCertificateClient.COMPONENT_NAME);
+
+    return HddsUtils.createDir(metadataDir
+        + File.separator
+        + DNCertificateClient.COMPONENT_NAME
+        + File.separator
+        + OzoneConsts.CRL_DB_DIRECTORY_NAME);
+  }
+
+  @Override
+  public DBColumnFamilyDefinition[] getColumnFamilies() {
+    return new DBColumnFamilyDefinition[] {PENDING_CRLS, CRL_SEQUENCE_ID};
+  }
+}

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/datanode/metadata/DatanodeCRLStore.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.datanode.metadata;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
+import org.apache.hadoop.hdds.utils.db.DBStore;
+import org.apache.hadoop.hdds.utils.db.Table;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Generic interface for data stores for Datanode.
+ * This is similar to the OMMetadataStore class,
+ * where we write classes into some underlying storage system.
+ */
+public interface DatanodeCRLStore {
+
+  /**
+   * Start metadata manager.
+   *
+   * @param configuration - Configuration
+   * @throws IOException - Unable to start metadata store.
+   */
+  void start(OzoneConfiguration configuration) throws IOException;
+
+  /**
+   * Stop metadata manager.
+   */
+  void stop() throws Exception;
+
+  /**
+   * Get metadata store.
+   *
+   * @return metadata store.
+   */
+  @VisibleForTesting
+  DBStore getStore();
+
+  /**
+   * A table to store the latest processed CRL sequence Id.
+   * @return Table
+   */
+  Table<String, Long> getCRLSequenceIdTable();
+
+  /**
+   * A table to store all the pending CRLs for future revocation of
+   * certificates.
+   * @return Table
+   */
+  Table<Long, CRLInfo> getPendingCRLsTable();
+
+  /**
+   * Returns the latest processed CRL Sequence ID.
+   * @return CRL Sequence ID.
+   * @throws IOException on error.
+   */
+  Long getLatestCRLSequenceID() throws IOException;
+
+  /**
+   * Return a list of CRLs that are pending revocation.
+   * @return a list of CRLInfo.
+   * @throws IOException on error.
+   */
+  List<CRLInfo> getPendingCRLs() throws IOException;
+
+}