diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java index f07539abdca5..5d219a8f79dd 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java @@ -90,7 +90,11 @@ public boolean isForce() { public String toString() { StringBuilder sb = new StringBuilder(); sb.append(getType()) - .append(": containerID: ").append(getContainerID()) + .append(": cmdID: ").append(getId()) + .append(", encodedToken: \"").append(getEncodedToken()).append("\"") + .append(", term: ").append(getTerm()) + .append(", deadlineMsSinceEpoch: ").append(getDeadline()) + .append(", containerID: ").append(getContainerID()) .append(", pipelineID: ").append(getPipelineID()) .append(", force: ").append(force); return sb.toString(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ClosePipelineCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ClosePipelineCommand.java index 1f75bc375ff1..634b01ba34ef 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ClosePipelineCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ClosePipelineCommand.java @@ -70,4 +70,16 @@ public static ClosePipelineCommand getFromProtobuf( public PipelineID getPipelineID() { return pipelineID; } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getType()) + .append(": cmdID: ").append(getId()) + .append(", encodedToken: \"").append(getEncodedToken()).append("\"") + .append(", term: ").append(getTerm()) + .append(", deadlineMsSinceEpoch: ").append(getDeadline()) + .append(", pipelineID: ").append(getPipelineID()); + return sb.toString(); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CreatePipelineCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CreatePipelineCommand.java index 44a883bb9024..a2791b0d47e9 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CreatePipelineCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CreatePipelineCommand.java @@ -155,4 +155,20 @@ public ReplicationType getReplicationType() { public ReplicationFactor getFactor() { return factor; } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getType()) + .append(": cmdID: ").append(getId()) + .append(", encodedToken: \"").append(getEncodedToken()).append("\"") + .append(", term: ").append(getTerm()) + .append(", deadlineMsSinceEpoch: ").append(getDeadline()) + .append(", pipelineID: ").append(getPipelineID()) + .append(", replicationFactor: ").append(factor) + .append(", replicationType: ").append(type) + .append(", nodelist: ").append(nodelist) + .append(", priorityList: ").append(priorityList); + return sb.toString(); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlocksCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlocksCommand.java index 03a876cee349..fbaf5175828c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlocksCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlocksCommand.java @@ -68,4 +68,27 @@ public DeleteBlocksCommandProto getProto() { .setCmdId(getId()) .addAllDeletedBlocksTransactions(blocksTobeDeleted).build(); } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getType()) + .append(": cmdID: ").append(getId()) + .append(", encodedToken: \"").append(getEncodedToken()).append("\"") + .append(", term: ").append(getTerm()) + .append(", deadlineMsSinceEpoch: ").append(getDeadline()) + .append(", deletedBlocksTransaction: ["); + for (DeletedBlocksTransaction txn : blocksTobeDeleted) { + sb.append("{ txnID: ").append(txn.getTxID()) + .append(", containerID: ").append(txn.getContainerID()) + .append(", deleteBlockCount: ").append(txn.getLocalIDCount()) + .append(", count: ").append(txn.getCount()) + .append("}, "); + } + if (!blocksTobeDeleted.isEmpty()) { + sb.delete(sb.length() - 2, sb.length()); + } + sb.append("]"); + return sb.toString(); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteContainerCommand.java index 6b76803dbf8a..fffdf91a5e13 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteContainerCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteContainerCommand.java @@ -110,7 +110,11 @@ public int getReplicaIndex() { public String toString() { StringBuilder sb = new StringBuilder(); sb.append(getType()) - .append(": containerID: ").append(getContainerID()) + .append(": cmdID: ").append(getId()) + .append(", encodedToken: \"").append(getEncodedToken()).append("\"") + .append(", term: ").append(getTerm()) + .append(", deadlineMsSinceEpoch: ").append(getDeadline()) + .append(", containerID: ").append(getContainerID()) .append(", replicaIndex: ").append(getReplicaIndex()) .append(", force: ").append(force); return sb.toString(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/FinalizeNewLayoutVersionCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/FinalizeNewLayoutVersionCommand.java index 74af7da1b57d..1f580049f168 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/FinalizeNewLayoutVersionCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/FinalizeNewLayoutVersionCommand.java @@ -77,4 +77,17 @@ public static FinalizeNewLayoutVersionCommand getFromProtobuf( finalizeProto.getFinalizeNewLayoutVersion(), finalizeProto.getDataNodeLayoutVersion(), finalizeProto.getCmdId()); } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getType()) + .append(": cmdID: ").append(getId()) + .append(", encodedToken: \"").append(getEncodedToken()).append("\"") + .append(", term: ").append(getTerm()) + .append(", deadlineMsSinceEpoch: ").append(getDeadline()) + .append(", finalizeUpgrade: ").append(finalizeUpgrade) + .append(", layoutInfo: ").append(layoutInfo); + return sb.toString(); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconstructECContainersCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconstructECContainersCommand.java index ada80c980f64..b689b049618c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconstructECContainersCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconstructECContainersCommand.java @@ -132,7 +132,11 @@ public ECReplicationConfig getEcReplicationConfig() { public String toString() { StringBuilder sb = new StringBuilder(); sb.append(getType()) - .append(": containerID: ").append(containerID) + .append(": cmdID: ").append(getId()) + .append(", encodedToken: \"").append(getEncodedToken()).append("\"") + .append(", term: ").append(getTerm()) + .append(", deadlineMsSinceEpoch: ").append(getDeadline()) + .append(", containerID: ").append(containerID) .append(", replicationConfig: ").append(ecReplicationConfig) .append(", sources: [").append(getSources().stream() .map(a -> a.dnDetails diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/RefreshVolumeUsageCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/RefreshVolumeUsageCommand.java index f009a0533fe1..c4fa2b5069b9 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/RefreshVolumeUsageCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/RefreshVolumeUsageCommand.java @@ -54,4 +54,15 @@ public static RefreshVolumeUsageCommand getFromProtobuf( Preconditions.checkNotNull(refreshVolumeUsageProto); return new RefreshVolumeUsageCommand(); } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getType()) + .append(": cmdID: ").append(getId()) + .append(", encodedToken: \"").append(getEncodedToken()).append("\"") + .append(", term: ").append(getTerm()) + .append(", deadlineMsSinceEpoch: ").append(getDeadline()); + return sb.toString(); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java index 5a8dc378cb53..2fd5f37adbb2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReplicateContainerCommand.java @@ -164,7 +164,11 @@ public ReplicationCommandPriority getPriority() { public String toString() { StringBuilder sb = new StringBuilder(); sb.append(getType()); - sb.append(": containerId=").append(getContainerID()); + sb.append(": cmdID: ").append(getId()) + .append(", encodedToken: \"").append(getEncodedToken()).append("\"") + .append(", term: ").append(getTerm()) + .append(", deadlineMsSinceEpoch: ").append(getDeadline()); + sb.append(", containerId=").append(getContainerID()); sb.append(", replicaIndex=").append(getReplicaIndex()); if (targetDatanode != null) { sb.append(", targetNode=").append(targetDatanode); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReregisterCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReregisterCommand.java index 6aa0554e1026..783e13ae1c6a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReregisterCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReregisterCommand.java @@ -55,4 +55,15 @@ public ReregisterCommandProto getProto() { .newBuilder() .build(); } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getType()) + .append(": cmdID: ").append(getId()) + .append(", encodedToken: \"").append(getEncodedToken()).append("\"") + .append(", term: ").append(getTerm()) + .append(", deadlineMsSinceEpoch: ").append(getDeadline()); + return sb.toString(); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/SetNodeOperationalStateCommand.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/SetNodeOperationalStateCommand.java index 2d3896ff8fdf..19a04e051aa8 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/SetNodeOperationalStateCommand.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/SetNodeOperationalStateCommand.java @@ -86,4 +86,17 @@ public static SetNodeOperationalStateCommand getFromProtobuf( cmdProto.getNodeOperationalState(), cmdProto.getStateExpiryEpochSeconds()); } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getType()) + .append(": cmdID: ").append(getId()) + .append(", encodedToken: \"").append(getEncodedToken()).append("\"") + .append(", term: ").append(getTerm()) + .append(", deadlineMsSinceEpoch: ").append(getDeadline()) + .append(", opState: ").append(opState) + .append(", stateExpiryEpochSeconds: ").append(stateExpiryEpochSeconds); + return sb.toString(); + } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java index 5a4dc505d84f..84b4e026f5ae 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeHeartbeatDispatcher.java @@ -200,6 +200,9 @@ public List dispatch(SCMHeartbeatRequestProto heartbeat) { } } } + if (LOG.isDebugEnabled()) { + LOG.debug("Heartbeat dispatched: datanode=" + datanodeDetails.getUuid() + ", Commands= " + commands); + } return commands; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java index b03f737b3406..ed15bd1ac01f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java @@ -280,6 +280,30 @@ public static SCMRegisteredResponseProto getRegisteredResponse( return cmd.getProtoBufMessage(); } + private String constructCommandAuditMap(List cmds) { + StringBuilder auditMap = new StringBuilder(); + auditMap.append("["); + for (SCMCommandProto cmd : cmds) { + if (cmd.getCommandType().equals(deleteBlocksCommand)) { + auditMap.append("commandType: ").append(cmd.getCommandType()); + auditMap.append(" deleteTransactionsCount: ") + .append(cmd.getDeleteBlocksCommandProto().getDeletedBlocksTransactionsCount()); + auditMap.append(" cmdID: ").append(cmd.getDeleteBlocksCommandProto().getCmdId()); + auditMap.append(" encodedToken: \"").append(cmd.getEncodedToken()).append("\""); + auditMap.append(" deadlineMsSinceEpoch: ").append(cmd.getDeadlineMsSinceEpoch()); + } else { + auditMap.append(cmd); + } + auditMap.append(", "); + } + int len = auditMap.length(); + if (len > 2) { + auditMap.delete(len - 2, len); + } + auditMap.append("]"); + return auditMap.toString(); + } + @Override public SCMHeartbeatResponseProto sendHeartbeat( SCMHeartbeatRequestProto heartbeat) throws IOException, TimeoutException { @@ -291,7 +315,7 @@ public SCMHeartbeatResponseProto sendHeartbeat( boolean auditSuccess = true; Map auditMap = Maps.newHashMap(); auditMap.put("datanodeUUID", heartbeat.getDatanodeDetails().getUuid()); - auditMap.put("command", flatten(cmdResponses.toString())); + auditMap.put("command", flatten(constructCommandAuditMap(cmdResponses))); term.ifPresent(t -> auditMap.put("term", String.valueOf(t))); try { SCMHeartbeatResponseProto.Builder builder =