Skip to content

HDDS-12175. Audit logs in SCM shouldn't print delete txns #7805

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nandakumar131 merged 12 commits into from
Feb 12, 2025
Merged

HDDS-12175. Audit logs in SCM shouldn't print delete txns #7805

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
ab716c3
HDDS-12175. Audit logs in SCM shouldn't print delete txns
Feb 4, 2025
66c6469
Fix format of commands list
Feb 5, 2025
252936d
Fix format of commands list
Feb 5, 2025
61d476b
improve code len
Feb 6, 2025
5040aec
improve code len
Feb 6, 2025
9415045
Add debug log for txn info
Feb 7, 2025
4f183d5
Add debug log for txn info
Feb 7, 2025
de71a08
Format output
Feb 7, 2025
c78fcb6
Format output
Feb 7, 2025
e62c92c
Override toString methods of all SCMCommands
Feb 11, 2025
8bd4958
Format deleteblocksCommand toString
Feb 12, 2025
77b6455
Remove duplicate lines
Feb 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions ...r-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
Expand All @@ -41,6 +42,7 @@
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.server.events.EventPublisher;
import org.apache.hadoop.hdds.server.events.IEventInfo;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;

Expand All @@ -53,6 +55,7 @@
import java.util.Map;
import java.util.UUID;

import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type.deleteBlocksCommand;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_ACTIONS;
import static org.apache.hadoop.hdds.scm.events.SCMEvents.CONTAINER_REPORT;
import static org.apache.hadoop.hdds.scm.events.SCMEvents
Expand Down Expand Up @@ -200,6 +203,34 @@ public List<SCMCommand> dispatch(SCMHeartbeatRequestProto heartbeat) {
}
}
}
if (LOG.isDebugEnabled()) {
StringBuilder allCommands = new StringBuilder();
for (SCMCommand cmd : commands) {
allCommands.append("cmdID: ").append(cmd.getId());
allCommands.append(" encodedToken: \"").append(cmd.getEncodedToken()).append("\"");
allCommands.append(" term: ").append(cmd.getTerm());
allCommands.append(" deadlineMsSinceEpoch: ").append(cmd.getDeadline());
if (cmd.getType().equals(deleteBlocksCommand)) {
DeleteBlocksCommand delCmd = (DeleteBlocksCommand) cmd;
allCommands.append(" deleteBlocksTransactions: {");
for (DeletedBlocksTransaction txn : delCmd.blocksTobeDeleted()) {
allCommands.append(" txnId:").append(txn.getTxID());
allCommands.append(" containerID:").append(txn.getContainerID());
allCommands.append(" deleteBlockCount:").append(txn.getLocalIDCount());
allCommands.append(" count:").append(txn.getCount());
allCommands.append(",");
}
allCommands.deleteCharAt(allCommands.length() - 1);
allCommands.append(" }");
} else {
allCommands.append(" ").append(cmd.getProto());
}
allCommands.append(", ");
}
int len = allCommands.length();
allCommands.delete(len - 2, len);
LOG.debug("Heartbeat dispatched: datanode=" + datanodeDetails.getUuid() + ", Commands= [" + allCommands + "]");
}

return commands;
}
Expand Down
26 changes: 25 additions & 1 deletion ...server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,30 @@ public static SCMRegisteredResponseProto getRegisteredResponse(
return cmd.getProtoBufMessage();
}

private String constructCommandAuditMap(List<SCMCommandProto> cmds) {
StringBuilder auditMap = new StringBuilder();
auditMap.append("[");
for (SCMCommandProto cmd : cmds) {
if (cmd.getCommandType().equals(deleteBlocksCommand)) {
auditMap.append("commandType: ").append(cmd.getCommandType());
auditMap.append(" deleteTransactionsCount: ")
.append(cmd.getDeleteBlocksCommandProto().getDeletedBlocksTransactionsCount());
auditMap.append(" cmdID: ").append(cmd.getDeleteBlocksCommandProto().getCmdId());
auditMap.append(" encodedToken: \"").append(cmd.getEncodedToken()).append("\"");
auditMap.append(" deadlineMsSinceEpoch: ").append(cmd.getDeadlineMsSinceEpoch());
} else {
auditMap.append(cmd);
}
auditMap.append(", ");
}
int len = auditMap.length();
if (len > 2) {
auditMap.delete(len - 2, len);
}
auditMap.append("]");
return auditMap.toString();
}

@Override
public SCMHeartbeatResponseProto sendHeartbeat(
SCMHeartbeatRequestProto heartbeat) throws IOException, TimeoutException {
Expand All @@ -291,7 +315,7 @@ public SCMHeartbeatResponseProto sendHeartbeat(
boolean auditSuccess = true;
Map<String, String> auditMap = Maps.newHashMap();
auditMap.put("datanodeUUID", heartbeat.getDatanodeDetails().getUuid());
auditMap.put("command", flatten(cmdResponses.toString()));
auditMap.put("command", flatten(constructCommandAuditMap(cmdResponses)));
term.ifPresent(t -> auditMap.put("term", String.valueOf(t)));
try {
SCMHeartbeatResponseProto.Builder builder =
Expand Down