HDDS-1480. Prefer resolved datanode ip address over persisted ip address

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java

@@ -30,6 +30,8 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.nio.file.Paths;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
@@ -174,20 +176,35 @@ public static synchronized DatanodeDetails readDatanodeDetailsFrom(File path)
     if (!path.exists()) {
       throw new IOException("Datanode ID file not found.");
     }
+    DatanodeDetails datanodeDetails;
     try {
-      return DatanodeIdYaml.readDatanodeIdFile(path);
+      datanodeDetails = DatanodeIdYaml.readDatanodeIdFile(path);
     } catch (IOException e) {
       LOG.warn("Error loading DatanodeDetails yaml from {}",
           path.getAbsolutePath(), e);
       // Try to load as protobuf before giving up
       try (FileInputStream in = new FileInputStream(path)) {
-        return DatanodeDetails.getFromProtoBuf(
+        datanodeDetails = DatanodeDetails.getFromProtoBuf(
             HddsProtos.DatanodeDetailsProto.parseFrom(in));
       } catch (IOException io) {
         throw new IOException("Failed to parse DatanodeDetails from "
             + path.getAbsolutePath(), io);
       }
     }
+
+    try {
+      InetAddress inetAddress = InetAddress.getByName(datanodeDetails.getHostName());
+
+      if (!inetAddress.getHostAddress().equals(datanodeDetails.getIpAddress())) {
+        LOG.warn("Resolved IP address '{}' differs from the persisted IP address '{}' for the datanode '{}'",
+            inetAddress.getHostAddress(), datanodeDetails.getIpAddress(), datanodeDetails.getHostName());
+        datanodeDetails.setIpAddress(inetAddress.getHostAddress());
+      }
+    } catch (UnknownHostException e) {
+      LOG.warn("Failed to validate IP address for the datanode '{}'", datanodeDetails.getHostName(), e);
+    }
+
+    return datanodeDetails;
   }
 
   /**

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/helpers/TestContainerUtils.java

@@ -30,10 +30,12 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.mockito.MockedStatic;
 
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.net.InetAddress;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 
@@ -44,7 +46,11 @@
 import static org.apache.hadoop.hdds.HddsUtils.processForDebug;
 import static org.apache.hadoop.ozone.container.ContainerTestHelper.getDummyCommandRequestProto;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.when;
 
 /**
  * Test for {@link ContainerUtils}.
@@ -91,34 +97,49 @@ public void testTarName() throws IOException {
   public void testDatanodeIDPersistent(@TempDir File tempDir) throws Exception {
     // Generate IDs for testing
     DatanodeDetails id1 = randomDatanodeDetails();
-    id1.setPort(DatanodeDetails.newStandalonePort(1));
-    assertWriteRead(tempDir, id1);
-
-    // Add certificate serial  id.
-    id1.setCertSerialId("" + RandomUtils.nextLong());
-    assertWriteRead(tempDir, id1);
-
-    // Read should return an empty value if file doesn't exist
-    File nonExistFile = new File(tempDir, "non_exist.id");
-    assertThrows(IOException.class,
-        () -> ContainerUtils.readDatanodeDetailsFrom(nonExistFile));
-
-    // Read should fail if the file is malformed
-    File malformedFile = new File(tempDir, "malformed.id");
-    createMalformedIDFile(malformedFile);
-    assertThrows(IOException.class,
-        () -> ContainerUtils.readDatanodeDetailsFrom(malformedFile));
-
-    // Test upgrade scenario - protobuf file instead of yaml
-    File protoFile = new File(tempDir, "valid-proto.id");
-    try (FileOutputStream out = new FileOutputStream(protoFile)) {
-      HddsProtos.DatanodeDetailsProto proto = id1.getProtoBufMessage();
-      proto.writeTo(out);
+    try (MockedStatic<InetAddress> mockedStaticInetAddress = mockStatic(InetAddress.class)) {
+      InetAddress mockedInetAddress = mock(InetAddress.class);
+      mockedStaticInetAddress.when(() -> InetAddress.getByName(id1.getHostName()))
+          .thenReturn(mockedInetAddress);
+
+      // If persisted ip address is different from resolved ip address,
+      // DatanodeDetails should be set with the resolved ip address
+      when(mockedInetAddress.getHostAddress())
+          .thenReturn("127.0.0.1");
+      assertWriteReadWithChangedIpAddress(tempDir, id1);
+
+      when(mockedInetAddress.getHostAddress())
+          .thenReturn(id1.getIpAddress());
+
+      id1.setPort(DatanodeDetails.newStandalonePort(1));
+      assertWriteRead(tempDir, id1);
+
+      // Add certificate serial  id.
+      id1.setCertSerialId("" + RandomUtils.nextLong());
+      assertWriteRead(tempDir, id1);
+
+      // Read should return an empty value if file doesn't exist
+      File nonExistFile = new File(tempDir, "non_exist.id");
+      assertThrows(IOException.class,
+          () -> ContainerUtils.readDatanodeDetailsFrom(nonExistFile));
+
+      // Read should fail if the file is malformed
+      File malformedFile = new File(tempDir, "malformed.id");
+      createMalformedIDFile(malformedFile);
+      assertThrows(IOException.class,
+          () -> ContainerUtils.readDatanodeDetailsFrom(malformedFile));
+
+      // Test upgrade scenario - protobuf file instead of yaml
+      File protoFile = new File(tempDir, "valid-proto.id");
+      try (FileOutputStream out = new FileOutputStream(protoFile)) {
+        HddsProtos.DatanodeDetailsProto proto = id1.getProtoBufMessage();
+        proto.writeTo(out);
+      }
+      assertDetailsEquals(id1, ContainerUtils.readDatanodeDetailsFrom(protoFile));
+
+      id1.setInitialVersion(1);
+      assertWriteRead(tempDir, id1);
     }
-    assertDetailsEquals(id1, ContainerUtils.readDatanodeDetailsFrom(protoFile));
-
-    id1.setInitialVersion(1);
-    assertWriteRead(tempDir, id1);
   }
 
   private void assertWriteRead(@TempDir File tempDir,
@@ -133,6 +154,15 @@ private void assertWriteRead(@TempDir File tempDir,
     assertEquals(details.getCurrentVersion(), read.getCurrentVersion());
   }
 
+  private void assertWriteReadWithChangedIpAddress(@TempDir File tempDir,
+                                                   DatanodeDetails details) throws IOException {
+    // Write a single ID to the file and read it out
+    File file = new File(tempDir, "valid-values.id");
+    ContainerUtils.writeDatanodeDetailsTo(details, file, conf);
+    DatanodeDetails read = ContainerUtils.readDatanodeDetailsFrom(file);
+    assertNotEquals(details.toString(), read.toString());
+  }
+
   private void createMalformedIDFile(File malformedFile)
       throws IOException {
     DatanodeDetails id = randomDatanodeDetails();

hadoop-ozone/dist/src/main/compose/ozone-om-prepare/docker-compose.yaml

@@ -49,6 +49,7 @@ x-om:
 services:
   dn1:
     <<: *datanode
+    hostname: dn1
     networks:
      net:
        ipv4_address: 10.9.0.11
@@ -57,6 +58,7 @@ services:
       - ../..:${OZONE_DIR}
   dn2:
     <<: *datanode
+    hostname: dn2
     networks:
      net:
        ipv4_address: 10.9.0.12
@@ -65,6 +67,7 @@ services:
       - ../..:${OZONE_DIR}
   dn3:
     <<: *datanode
+    hostname: dn3
     networks:
      net:
        ipv4_address: 10.9.0.13