Skip to content

HDDS-12553. ozone admin container list should output real JSON array #8050

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 8 commits into from
Mar 26, 2025
Merged

HDDS-12553. ozone admin container list should output real JSON array #8050

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ec02348
HDDS-12553. Support ozone admin container list to print list in Json …
kerneltime Mar 11, 2025
f189a0a
HDDS-12553. Use SequenceWriter for JSON array output in container list
kerneltime Mar 12, 2025
74b2e6b
HDDS-12555. Change container list default output format to JSON array
kerneltime Mar 14, 2025
bbbe43c
HDDS-12555. Change container list default output format to JSON array
kerneltime Mar 14, 2025
f93eb1e
Deal with sequence writer closing System.out and not printing the fin…
kerneltime Mar 20, 2025
e34b38e
Address checkstyle and errors
kerneltime Mar 20, 2025
c7d086b
Fix container count robot test
kerneltime Mar 20, 2025
905852a
Fix the issues caused by std error output for successful CLI runs and…
kerneltime Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 98 additions & 19 deletions hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,11 @@ public class ListSubcommand extends ScmSubcommand {
description = "Container replication (ONE, THREE for Ratis, " +
"rs-6-3-1024k for EC)")
private String replication;

@Option(names = {"--json"},
description = "Output the entire list in JSON array format",
defaultValue = "false")
private boolean jsonFormat;

private static final ObjectWriter WRITER;

Expand All @@ -90,11 +95,35 @@ public class ListSubcommand extends ScmSubcommand {
}


private void outputContainerInfo(ContainerInfo containerInfo)
throws IOException {
// Print container report info.
private void outputContainerInfo(ContainerInfo containerInfo) throws IOException {
// Original behavior - just print the container JSON
System.out.println(WRITER.writeValueAsString(containerInfo));
}

private void outputContainerInfoAsJsonMember(ContainerInfo containerInfo, boolean isFirst,
boolean isLast) throws IOException {
// JSON array format with proper brackets and commas
if (isFirst) {
// Start of array
System.out.print("[");
}

// Print the container JSON
System.out.print(WRITER.writeValueAsString(containerInfo));

if (!isLast) {
// Add comma between elements
System.out.print(",");
}

if (isLast) {
// End of array
System.out.println("]");
} else {
// Add newline for readability
System.out.println();
}
}

@Override
public void execute(ScmClient scmClient) throws IOException {
Expand Down Expand Up @@ -124,8 +153,20 @@ public void execute(ScmClient scmClient) throws IOException {
count = maxCountAllowed;
}
containerListAndTotalCount = scmClient.listContainer(startId, count, state, type, repConfig);
for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) {
outputContainerInfo(container);

int totalSize = containerListAndTotalCount.getContainerInfoList().size();

if (jsonFormat) {
// JSON array format
for (int i = 0; i < totalSize; i++) {
ContainerInfo container = containerListAndTotalCount.getContainerInfoList().get(i);
outputContainerInfoAsJsonMember(container, i == 0, i == totalSize - 1);
}
} else {
// Original format - one JSON object per line
for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) {
outputContainerInfo(container);
}
}

if (containerListAndTotalCount.getTotalCount() > count) {
Expand All @@ -138,20 +179,58 @@ public void execute(ScmClient scmClient) throws IOException {
int batchSize = (count > 0) ? count : maxCountAllowed;
long currentStartId = startId;
int fetchedCount;

do {
// Fetch containers in batches of 'batchSize'
containerListAndTotalCount = scmClient.listContainer(currentStartId, batchSize, state, type, repConfig);
fetchedCount = containerListAndTotalCount.getContainerInfoList().size();

for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) {
outputContainerInfo(container);
}

if (fetchedCount > 0) {
currentStartId = containerListAndTotalCount.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1;
}
} while (fetchedCount > 0);

if (jsonFormat) {
// JSON array format for all containers
boolean isFirstContainer = true;

// Start JSON array
System.out.print("[");

do {
// Fetch containers in batches of 'batchSize'
containerListAndTotalCount = scmClient.listContainer(currentStartId, batchSize, state, type, repConfig);
fetchedCount = containerListAndTotalCount.getContainerInfoList().size();

for (int i = 0; i < fetchedCount; i++) {
ContainerInfo container = containerListAndTotalCount.getContainerInfoList().get(i);

// Only the first container overall doesn't need a preceding comma
if (!isFirstContainer) {
System.out.print(",");
System.out.println();
}

// Print the container JSON
System.out.print(WRITER.writeValueAsString(container));
isFirstContainer = false;
}

if (fetchedCount > 0) {
currentStartId =
containerListAndTotalCount.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1;
}
} while (fetchedCount > 0);

// Close the JSON array
System.out.println("]");
} else {
// Original format - one JSON object per line
do {
// Fetch containers in batches of 'batchSize'
containerListAndTotalCount = scmClient.listContainer(currentStartId, batchSize, state, type, repConfig);
fetchedCount = containerListAndTotalCount.getContainerInfoList().size();

for (ContainerInfo container : containerListAndTotalCount.getContainerInfoList()) {
outputContainerInfo(container);
}

if (fetchedCount > 0) {
currentStartId =
containerListAndTotalCount.getContainerInfoList().get(fetchedCount - 1).getContainerID() + 1;
}
} while (fetchedCount > 0);
}
}
}
}
27 changes: 27 additions & 0 deletions hadoop-ozone/dist/src/main/smoketest/admincli/container.robot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,33 @@ List all containers from a particular container ID
${output} = Execute ozone admin container list --all --start 1
Should contain ${output} OPEN

List containers in JSON array format
${output} = Execute ozone admin container list --json | jq -r '.'
Should Start With ${output} [
Should Contain ${output} containerID
Should End With ${output} ]

List all containers in JSON array format
${output} = Execute ozone admin container list --all --json | jq -r '.'
Should Start With ${output} [
Should Contain ${output} containerID
Should End With ${output} ]

List containers with state in JSON array format
${output} = Execute ozone admin container list --state=OPEN --json | jq -r '.'
Should Start With ${output} [
Should Contain ${output} OPEN
Should Not Contain ${output} CLOSED
Should End With ${output} ]

List containers with count in JSON array format
${output} = Execute ozone admin container list --count 5 --json | jq -r '.'
Should Start With ${output} [
Should Contain ${output} containerID
Should End With ${output} ]
${count} = Execute echo '${output}' | jq -r 'length'
Should Be True ${count} <= 5

Close container
${container} = Execute ozone admin container list --state OPEN | jq -r 'select(.replicationConfig.replicationFactor == "THREE") | .containerID' | head -1
Execute ozone admin container close "${container}"
Expand Down