From 77ec894ec6a17718fd82b1160273e3221abeea8b Mon Sep 17 00:00:00 2001 From: Symious Date: Mon, 24 Feb 2025 18:03:13 +0800 Subject: [PATCH 1/4] HDDS-12410. Add detailed block info for ALLOCATE_BLOCK audit log --- .../hadoop/hdds/scm/server/SCMBlockProtocolServer.java | 5 +++++ .../hdds/scm/server/TestSCMBlockProtocolServer.java | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 4c42bff28c55..2272743ae199 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -42,6 +42,7 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.hdds.client.BlockID; @@ -226,6 +227,10 @@ public List allocateBlock( } auditMap.put("allocated", String.valueOf(blocks.size())); + String blockIDs = blocks.stream() + .map(block -> block.getBlockID().toString()) + .collect(Collectors.joining(", ", "[", "]")); + auditMap.put("blocks", blockIDs); if (blocks.size() < num) { AUDIT.logWriteFailure(buildAuditMessageForFailure( diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java index d94c7e5a119f..87fe877f8a42 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_LEVEL; import static org.apache.hadoop.ozone.OzoneConsts.MB; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -62,10 +63,12 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocolServerSideTranslatorPB; import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics; +import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.StaticMapping; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.container.common.SCMTestUtils; +import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -306,10 +309,15 @@ void testAllocateBlockWithClientMachine() throws IOException { final long blockSize = 128 * MB; final int numOfBlocks = 5; + GenericTestUtils.LogCapturer logs = GenericTestUtils.LogCapturer.captureLogs(Server.AUDITLOG); List allocatedBlocks = server.allocateBlock( blockSize, numOfBlocks, replicationConfig, "o", new ExcludeList(), clientAddress); assertEquals(numOfBlocks, allocatedBlocks.size()); + // Assert if auth was successful via Kerberos + System.out.println("YYYY" + logs.getOutput()); + assertThat(logs.getOutput()).doesNotContain( + "conID"); for (AllocatedBlock allocatedBlock: allocatedBlocks) { List nodesInOrder = allocatedBlock.getPipeline().getNodesInOrder(); From d2981bf1aff4194ec9b120a5fbfc7f84657b80fe Mon Sep 17 00:00:00 2001 From: Symious Date: Tue, 25 Feb 2025 17:56:39 +0800 Subject: [PATCH 2/4] HDDS-12410. Remove unused code --- .../hdds/scm/server/TestSCMBlockProtocolServer.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java index 87fe877f8a42..d94c7e5a119f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMBlockProtocolServer.java @@ -21,7 +21,6 @@ import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_LEVEL; import static org.apache.hadoop.ozone.OzoneConsts.MB; -import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -63,12 +62,10 @@ import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocolServerSideTranslatorPB; import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics; -import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.StaticMapping; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.common.BlockGroup; import org.apache.hadoop.ozone.container.common.SCMTestUtils; -import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -309,15 +306,10 @@ void testAllocateBlockWithClientMachine() throws IOException { final long blockSize = 128 * MB; final int numOfBlocks = 5; - GenericTestUtils.LogCapturer logs = GenericTestUtils.LogCapturer.captureLogs(Server.AUDITLOG); List allocatedBlocks = server.allocateBlock( blockSize, numOfBlocks, replicationConfig, "o", new ExcludeList(), clientAddress); assertEquals(numOfBlocks, allocatedBlocks.size()); - // Assert if auth was successful via Kerberos - System.out.println("YYYY" + logs.getOutput()); - assertThat(logs.getOutput()).doesNotContain( - "conID"); for (AllocatedBlock allocatedBlock: allocatedBlocks) { List nodesInOrder = allocatedBlock.getPipeline().getNodesInOrder(); From 180f2966df83a32a61cd216167649b74db12a64c Mon Sep 17 00:00:00 2001 From: Symious <14933944+symious@users.noreply.github.com> Date: Mon, 3 Mar 2025 13:40:39 +0800 Subject: [PATCH 3/4] Apply suggestions from code review Co-authored-by: Nandakumar Vadivelu --- .../apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 2272743ae199..33f6e3ca88ce 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -227,10 +227,10 @@ public List allocateBlock( } auditMap.put("allocated", String.valueOf(blocks.size())); - String blockIDs = blocks.stream() + String blockIDs = blocks.stream().limit(10) .map(block -> block.getBlockID().toString()) .collect(Collectors.joining(", ", "[", "]")); - auditMap.put("blocks", blockIDs); + auditMap.put("sample blocks", blockIDs); if (blocks.size() < num) { AUDIT.logWriteFailure(buildAuditMessageForFailure( From e2067f75898ad166b87988e045b7f753df22aa0a Mon Sep 17 00:00:00 2001 From: Symious Date: Tue, 4 Mar 2025 09:09:35 +0800 Subject: [PATCH 4/4] HDDS-12410. Remove space in key --- .../apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java index 33f6e3ca88ce..4ee10b70b3a8 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java @@ -230,7 +230,7 @@ public List allocateBlock( String blockIDs = blocks.stream().limit(10) .map(block -> block.getBlockID().toString()) .collect(Collectors.joining(", ", "[", "]")); - auditMap.put("sample blocks", blockIDs); + auditMap.put("sampleBlocks", blockIDs); if (blocks.size() < num) { AUDIT.logWriteFailure(buildAuditMessageForFailure(