Skip to content

HDDS-10091. TestOzoneDebugShell fails with FILE_SYSTEM_OPTIMIZED bucket layout #8227

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 4 commits into from
Apr 6, 2025
Merged

HDDS-10091. TestOzoneDebugShell fails with FILE_SYSTEM_OPTIMIZED bucket layout #8227

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
afbd320
HDDS-10091. Updating column family for FILE_SYSTEM_OPTIMIZED
priyeshkaratha Apr 3, 2025
12c03cd
ParameterizedTest to check all layouts
priyeshkaratha Apr 4, 2025
4ec0394
addressing review comments
priyeshkaratha Apr 5, 2025
377178d
Using running OM instead of mocking
priyeshkaratha Apr 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 17 additions & 12 deletions ...one/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneDebugShell.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.apache.hadoop.ozone.debug.OzoneDebug;
import org.apache.hadoop.ozone.debug.ldb.RDBParser;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
Expand All @@ -58,9 +59,9 @@
import org.apache.ozone.test.NonHATests;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.ValueSource;
import picocli.CommandLine;

Expand All @@ -72,11 +73,13 @@ public abstract class TestOzoneDebugShell implements NonHATests.TestCase {

private OzoneClient client;
private OzoneDebug ozoneDebugShell;
private OMMetadataManager omMetadataManager;

@BeforeEach
void init() throws Exception {
ozoneDebugShell = new OzoneDebug();
client = cluster().newClient();
omMetadataManager = cluster().getOzoneManager().getMetadataManager();
}

@AfterEach
Expand All @@ -91,7 +94,7 @@ public void testChunkInfoCmdBeforeAfterCloseContainer(boolean isEcKey) throws Ex
final String bucketName = UUID.randomUUID().toString();
final String keyName = UUID.randomUUID().toString();

writeKey(volumeName, bucketName, keyName, isEcKey);
writeKey(volumeName, bucketName, keyName, isEcKey, BucketLayout.FILE_SYSTEM_OPTIMIZED);

int exitCode = runChunkInfoCommand(volumeName, bucketName, keyName);
assertEquals(0, exitCode);
Expand All @@ -102,18 +105,21 @@ public void testChunkInfoCmdBeforeAfterCloseContainer(boolean isEcKey) throws Ex
assertEquals(0, exitCode);
}

@Test
public void testChunkInfoVerifyPathsAreDifferent() throws Exception {
@ParameterizedTest
@EnumSource
public void testChunkInfoVerifyPathsAreDifferent(BucketLayout layout) throws Exception {
final String volumeName = UUID.randomUUID().toString();
final String bucketName = UUID.randomUUID().toString();
final String keyName = UUID.randomUUID().toString();
writeKey(volumeName, bucketName, keyName, false);
writeKey(volumeName, bucketName, keyName, false, layout);
int exitCode = runChunkInfoAndVerifyPaths(volumeName, bucketName, keyName);
assertEquals(0, exitCode);
}

@Test
public void testLdbCliForOzoneSnapshot() throws Exception {
@ParameterizedTest
@EnumSource
public void testLdbCliForOzoneSnapshot(BucketLayout layout) throws Exception {
String columnFamily = omMetadataManager.getKeyTable(layout).getName();
StringWriter stdout = new StringWriter();
PrintWriter pstdout = new PrintWriter(stdout);
CommandLine cmd = new CommandLine(new RDBParser())
Expand All @@ -122,7 +128,7 @@ public void testLdbCliForOzoneSnapshot() throws Exception {
final String bucketName = UUID.randomUUID().toString();
final String keyName = UUID.randomUUID().toString();

writeKey(volumeName, bucketName, keyName, false);
writeKey(volumeName, bucketName, keyName, false, layout);

String snapshotName =
client.getObjectStore().createSnapshot(volumeName, bucketName, "snap1");
Expand All @@ -135,7 +141,7 @@ public void testLdbCliForOzoneSnapshot() throws Exception {
GenericTestUtils
.waitFor(() -> new File(snapshotCurrent).exists(), 1000, 120000);
String[] args =
new String[] {"--db=" + dbPath, "scan", "--cf", "keyTable"};
new String[] {"--db=" + dbPath, "scan", "--cf", columnFamily};
int exitCode = cmd.execute(args);
assertEquals(0, exitCode);
String cmdOut = stdout.toString();
Expand All @@ -149,17 +155,16 @@ private String getSnapshotDBPath(String checkPointDir) {
}

private void writeKey(String volumeName, String bucketName,
String keyName, boolean isEcKey) throws IOException {
String keyName, boolean isEcKey, BucketLayout layout) throws IOException {
ReplicationConfig repConfig;
if (isEcKey) {
repConfig = new ECReplicationConfig(3, 2);
} else {
repConfig = ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS,
ReplicationFactor.THREE);
}
// see HDDS-10091 for making this work with FILE_SYSTEM_OPTIMIZED layout
TestDataUtil.createVolumeAndBucket(client, volumeName, bucketName,
BucketLayout.LEGACY);
layout);
TestDataUtil.createKey(
client.getObjectStore().getVolume(volumeName).getBucket(bucketName),
keyName, repConfig, "test".getBytes(StandardCharsets.UTF_8));
Expand Down