HDDS-7817. Add HTTP and HTTPS ports to DatanodeDetails

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java

@@ -37,6 +37,8 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import org.apache.hadoop.hdds.upgrade.BelongsToHDDSLayoutVersion;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
 import org.apache.hadoop.ozone.ClientVersion;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -802,7 +804,13 @@ public static final class Port {
      */
     public enum Name {
       STANDALONE, RATIS, REST, REPLICATION, RATIS_ADMIN, RATIS_SERVER,
-      RATIS_DATASTREAM;
+      RATIS_DATASTREAM,
+      @BelongsToHDDSLayoutVersion(HDDSLayoutFeature
+          .WEBUI_PORTS_IN_DATANODEDETAILS)
+      HTTP,
+      @BelongsToHDDSLayoutVersion(HDDSLayoutFeature
+          .WEBUI_PORTS_IN_DATANODEDETAILS)
+      HTTPS;
 
       public static final Set<Name> ALL_PORTS = ImmutableSet.copyOf(
           Name.values());

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/BelongsToHDDSLayoutVersion.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.upgrade;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to mark a class that belongs to a specific HDDS Layout Version.
+ */
+@Target({ElementType.TYPE, ElementType.FIELD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface BelongsToHDDSLayoutVersion {
+  HDDSLayoutFeature value();
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/upgrade/HDDSLayoutFeature.java

@@ -35,7 +35,9 @@ public enum HDDSLayoutFeature implements LayoutFeature {
   ERASURE_CODED_STORAGE_SUPPORT(3, "Ozone version with built in support for"
       + " Erasure Coded block data storage."),
   DATANODE_SCHEMA_V3(4, "Datanode RocksDB Schema Version 3 (one rocksdb " +
-      "per disk)");
+      "per disk)"),
+  WEBUI_PORTS_IN_DATANODEDETAILS(5, "Adding HTTP and HTTPS ports " +
+      "to DatanodeDetails.");
 
   //////////////////////////////  //////////////////////////////
 

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java

@@ -44,6 +44,7 @@
 import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
 import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClient;
 import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateSignRequest;
+import org.apache.hadoop.hdds.server.http.HttpConfig;
 import org.apache.hadoop.hdds.server.http.RatisDropwizardExports;
 import org.apache.hadoop.hdds.tracing.TracingUtil;
 import org.apache.hadoop.hdds.utils.HddsServerUtil;
@@ -67,6 +68,8 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
+import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.HTTP;
+import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.HTTPS;
 import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY;
 import static org.apache.hadoop.ozone.conf.OzoneServiceConfig.DEFAULT_SHUTDOWN_HOOK_PRIORITY;
 import static org.apache.hadoop.ozone.common.Storage.StorageState.INITIALIZED;
@@ -276,6 +279,15 @@ public void start() {
       try {
         httpServer = new HddsDatanodeHttpServer(conf);
         httpServer.start();
+        HttpConfig.Policy policy = HttpConfig.getHttpPolicy(conf);
+        if (policy.isHttpEnabled()) {
+          datanodeDetails.setPort(DatanodeDetails.newPort(HTTP,
+                  httpServer.getHttpAddress().getPort()));
+        }
+        if (policy.isHttpsEnabled()) {
+          datanodeDetails.setPort(DatanodeDetails.newPort(HTTPS,
+                  httpServer.getHttpsAddress().getPort()));
+        }
       } catch (Exception ex) {
         LOG.error("HttpServer failed to start.", ex);
       }
@@ -420,7 +432,7 @@ private void persistDatanodeDetails(DatanodeDetails dnDetails)
     String idFilePath = HddsServerUtil.getDatanodeIdFilePath(conf);
     Preconditions.checkNotNull(idFilePath);
     File idFile = new File(idFilePath);
-    ContainerUtils.writeDatanodeDetailsTo(dnDetails, idFile);
+    ContainerUtils.writeDatanodeDetailsTo(dnDetails, idFile, conf);
   }
 
   /**

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java

@@ -141,7 +141,8 @@ public static String getContainerDbFileName(String containerName) {
    * @throws IOException when read/write error occurs
    */
   public static synchronized void writeDatanodeDetailsTo(
-      DatanodeDetails datanodeDetails, File path) throws IOException {
+      DatanodeDetails datanodeDetails, File path, ConfigurationSource conf)
+      throws IOException {
     if (path.exists()) {
       if (!path.delete() || !path.createNewFile()) {
         throw new IOException("Unable to overwrite the datanode ID file.");
@@ -152,7 +153,7 @@ public static synchronized void writeDatanodeDetailsTo(
         throw new IOException("Unable to create datanode ID directories.");
       }
     }
-    DatanodeIdYaml.createDatanodeIdFile(datanodeDetails, path);
+    DatanodeIdYaml.createDatanodeIdFile(datanodeDetails, path, conf);
   }
 
   /**

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DatanodeIdYaml.java

@@ -23,16 +23,21 @@
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
+import java.lang.reflect.Field;
 import java.nio.charset.StandardCharsets;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.UUID;
 
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.server.YamlUtils;
+import org.apache.hadoop.hdds.upgrade.BelongsToHDDSLayoutVersion;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
+import org.apache.hadoop.ozone.container.common.DatanodeLayoutStorage;
 import org.yaml.snakeyaml.DumperOptions;
 import org.yaml.snakeyaml.Yaml;
 
@@ -53,15 +58,17 @@ private DatanodeIdYaml() {
    * @param path            Path to datnode.id file
    */
   public static void createDatanodeIdFile(DatanodeDetails datanodeDetails,
-                                          File path) throws IOException {
+                                          File path,
+                                          ConfigurationSource conf)
+          throws IOException {
     DumperOptions options = new DumperOptions();
     options.setPrettyFlow(true);
     options.setDefaultFlowStyle(DumperOptions.FlowStyle.FLOW);
     Yaml yaml = new Yaml(options);
 
     try (Writer writer = new OutputStreamWriter(
         new FileOutputStream(path), StandardCharsets.UTF_8)) {
-      yaml.dump(getDatanodeDetailsYaml(datanodeDetails), writer);
+      yaml.dump(getDatanodeDetailsYaml(datanodeDetails, conf), writer);
     }
   }
 
@@ -219,11 +226,31 @@ public void setCurrentVersion(int version) {
   }
 
   private static DatanodeDetailsYaml getDatanodeDetailsYaml(
-      DatanodeDetails datanodeDetails) {
+      DatanodeDetails datanodeDetails, ConfigurationSource conf)
+      throws IOException {
+
+    DatanodeLayoutStorage datanodeLayoutStorage
+        = new DatanodeLayoutStorage(conf, datanodeDetails.getUuidString());
 
     Map<String, Integer> portDetails = new LinkedHashMap<>();
     if (!CollectionUtils.isEmpty(datanodeDetails.getPorts())) {
       for (DatanodeDetails.Port port : datanodeDetails.getPorts()) {
+        Field f = null;
+        try {
+          f = port.getName().getClass()
+              .getDeclaredField(port.getName().toString());
+        } catch (NoSuchFieldException e) {
+          e.printStackTrace();
+        }
+        if (f != null
+            && f.isAnnotationPresent(BelongsToHDDSLayoutVersion.class)) {
+          HDDSLayoutFeature layoutFeature
+              = f.getAnnotation(BelongsToHDDSLayoutVersion.class).value();
+          if (layoutFeature.layoutVersion() >
+              datanodeLayoutStorage.getLayoutVersion()) {
+            continue;
+          }
+        }
         portDetails.put(port.getName().toString(), port.getValue());
       }
     }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java

@@ -108,7 +108,7 @@ private void persistDatanodeDetails(DatanodeDetails dnDetails)
     String idFilePath = HddsServerUtil.getDatanodeIdFilePath(conf);
     Preconditions.checkNotNull(idFilePath);
     File idFile = new File(idFilePath);
-    ContainerUtils.writeDatanodeDetailsTo(dnDetails, idFile);
+    ContainerUtils.writeDatanodeDetailsTo(dnDetails, idFile, conf);
   }
 
   /**

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java

@@ -127,7 +127,7 @@ private void persistContainerDatanodeDetails() {
         .getDatanodeDetails();
     if (datanodeDetails != null) {
       try {
-        ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, idPath);
+        ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, idPath, conf);
       } catch (IOException ex) {
         // As writing DatanodeDetails in to datanodeid file failed, which is
         // a critical thing, so shutting down the state machine.

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java

@@ -218,7 +218,7 @@ public void testDatanodeStateContext() throws IOException,
         DatanodeDetails.Port.Name.STANDALONE,
         OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
     datanodeDetails.setPort(port);
-    ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, idPath);
+    ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, idPath, conf);
     try (DatanodeStateMachine stateMachine =
              new DatanodeStateMachine(datanodeDetails, conf, null, null,
                  null)) {

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestContainerUtils.java

@@ -18,13 +18,16 @@
 package org.apache.hadoop.ozone.container.common.helpers;
 
 import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.ByteStringConversion;
 import org.apache.hadoop.ozone.common.ChunkBuffer;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
@@ -49,6 +52,14 @@
  */
 public class TestContainerUtils {
 
+  private OzoneConfiguration conf;
+
+  @BeforeEach
+  void setup(@TempDir File dir) {
+    conf = new OzoneConfiguration();
+    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.toString());
+  }
+
   @Test
   public void redactsDataBuffers() {
     // GIVEN
@@ -112,11 +123,11 @@ public void testDatanodeIDPersistent(@TempDir File tempDir) throws Exception {
     assertWriteRead(tempDir, id1);
   }
 
-  private static void assertWriteRead(@TempDir File tempDir,
+  private void assertWriteRead(@TempDir File tempDir,
       DatanodeDetails details) throws IOException {
     // Write a single ID to the file and read it out
     File file = new File(tempDir, "valid-values.id");
-    ContainerUtils.writeDatanodeDetailsTo(details, file);
+    ContainerUtils.writeDatanodeDetailsTo(details, file, conf);
 
     DatanodeDetails read = ContainerUtils.readDatanodeDetailsFrom(file);
 
@@ -127,7 +138,7 @@ private static void assertWriteRead(@TempDir File tempDir,
   private void createMalformedIDFile(File malformedFile)
       throws IOException {
     DatanodeDetails id = randomDatanodeDetails();
-    ContainerUtils.writeDatanodeDetailsTo(id, malformedFile);
+    ContainerUtils.writeDatanodeDetailsTo(id, malformedFile, conf);
 
     try (FileOutputStream out = new FileOutputStream(malformedFile)) {
       out.write("malformed".getBytes(StandardCharsets.UTF_8));

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestDatanodeIdYaml.java

@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.ozone.container.common.helpers;
 
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
 import org.junit.jupiter.api.Test;
@@ -36,8 +38,10 @@ class TestDatanodeIdYaml {
   void testWriteRead(@TempDir File dir) throws IOException {
     DatanodeDetails original = MockDatanodeDetails.randomDatanodeDetails();
     File file = new File(dir, "datanode.yaml");
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, dir.toString());
 
-    DatanodeIdYaml.createDatanodeIdFile(original, file);
+    DatanodeIdYaml.createDatanodeIdFile(original, file, conf);
     DatanodeDetails read = DatanodeIdYaml.readDatanodeIdFile(file);
 
     assertEquals(original, read);

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java

@@ -289,7 +289,8 @@ public void testAddHddsVolumeAfterFinalize() throws Exception {
 
     // Add a new HddsVolume. It should have DB created after DN restart.
     addHddsVolume();
-    restartDatanode(HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion(), true);
+    restartDatanode(HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion(),
+        false);
     for (StorageVolume vol:
         dsm.getContainer().getVolumeSet().getVolumesList()) {
       HddsVolume hddsVolume = (HddsVolume) vol;
@@ -323,7 +324,8 @@ public void testAddDbVolumeAfterFinalize() throws Exception {
 
     // Add a new DbVolume
     addDbVolume();
-    restartDatanode(HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion(), true);
+    restartDatanode(HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion(),
+        false);
 
     // HddsVolume should still use the rocksDB under it's volume
     DbVolume dbVolume = (DbVolume) dsm.getContainer().getDbVolumeSet()
@@ -352,7 +354,8 @@ public void testAddDbAndHddsVolumeAfterFinalize() throws Exception {
 
     addDbVolume();
     File newDataVolume = addHddsVolume();
-    restartDatanode(HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion(), true);
+    restartDatanode(HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion(),
+        false);
 
     DbVolume dbVolume = (DbVolume) dsm.getContainer().getDbVolumeSet()
         .getVolumesList().get(0);
@@ -424,7 +427,8 @@ public void testWrite(boolean enable, String expectedVersion)
     // Set SchemaV3 enable status
     conf.setBoolean(DatanodeConfiguration.CONTAINER_SCHEMA_V3_ENABLED,
         enable);
-    restartDatanode(HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion(), true);
+    restartDatanode(HDDSLayoutFeature.DATANODE_SCHEMA_V3.layoutVersion(),
+        false);
 
     // Write new data
     final long containerID2 = addContainer(pipeline);

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh

@@ -70,7 +70,7 @@ with_old_version_downgraded() {
 }
 
 with_new_version_finalized() {
-  _check_hdds_mlvs 4
+  _check_hdds_mlvs 5
   _check_om_mlvs 3
 
   validate old1