Skip to content

HDDS-12845. Reuse TestDataUtil.createKey in OzoneRpcClientTests #8274

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 3 commits into from
Apr 16, 2025
Merged

HDDS-12845. Reuse TestDataUtil.createKey in OzoneRpcClientTests #8274

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
17a536c
HDDS-12524. Reuse TestDataUtil.createKey in more tests
chiacyu Apr 13, 2025
eb21190
Fixed tests
chiacyu Apr 13, 2025
558fbb6
indent the functions
chiacyu Apr 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 71 additions & 132 deletions ...ntegration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1184,10 +1184,8 @@ public void testPutKeyWithReplicationConfig(String replicationValue,
ReplicationConfig replicationConfig =
new ECReplicationConfig(replicationValue);
if (isValidReplicationConfig) {
OzoneOutputStream out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, replicationConfig, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
replicationConfig, value.getBytes(UTF_8));
OzoneKey key = bucket.getKey(keyName);
assertEquals(keyName, key.getName());
try (OzoneInputStream is = bucket.readKey(keyName)) {
Expand Down Expand Up @@ -1217,11 +1215,9 @@ public void testPutKey() throws IOException {
for (int i = 0; i < 10; i++) {
String keyName = UUID.randomUUID().toString();

OzoneOutputStream out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, RATIS,
ONE, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
ReplicationConfig.fromTypeAndFactor(RATIS, ONE),
value.getBytes(UTF_8));
OzoneKey key = bucket.getKey(keyName);
assertEquals(keyName, key.getName());
try (OzoneInputStream is = bucket.readKey(keyName)) {
Expand Down Expand Up @@ -1995,10 +1991,9 @@ public void testValidateBlockLengthWithCommitKey() throws IOException {
String keyName = UUID.randomUUID().toString();

// create the initial key with size 0, write will allocate the first block.
OzoneOutputStream out = bucket.createKey(keyName, 0,
RATIS, ONE, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
ReplicationConfig.fromTypeAndFactor(RATIS, ONE),
value.getBytes(UTF_8));
OmKeyArgs.Builder builder = new OmKeyArgs.Builder();
builder.setVolumeName(volumeName).setBucketName(bucketName)
.setKeyName(keyName);
Expand Down Expand Up @@ -2030,11 +2025,9 @@ public void testPutKeyRatisOneNode() throws IOException {
for (int i = 0; i < 10; i++) {
String keyName = UUID.randomUUID().toString();

OzoneOutputStream out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, ReplicationType.RATIS,
ONE, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
ReplicationConfig.fromTypeAndFactor(RATIS, ONE),
value.getBytes(UTF_8));
OzoneKey key = bucket.getKey(keyName);
assertEquals(keyName, key.getName());
try (OzoneInputStream is = bucket.readKey(keyName)) {
Expand Down Expand Up @@ -2065,11 +2058,9 @@ public void testPutKeyRatisThreeNodes() throws IOException {
for (int i = 0; i < 10; i++) {
String keyName = UUID.randomUUID().toString();

OzoneOutputStream out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, ReplicationType.RATIS,
THREE, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
ReplicationConfig.fromTypeAndFactor(RATIS, THREE),
value.getBytes(UTF_8));
OzoneKey key = bucket.getKey(keyName);
assertEquals(keyName, key.getName());
try (OzoneInputStream is = bucket.readKey(keyName)) {
Expand Down Expand Up @@ -2107,11 +2098,9 @@ public void testPutKeyRatisThreeNodesParallel() throws IOException,
String keyName = UUID.randomUUID().toString();
String data = Arrays.toString(generateData(5 * 1024 * 1024,
(byte) RandomUtils.nextLong()));
OzoneOutputStream out = bucket.createKey(keyName,
data.getBytes(UTF_8).length, ReplicationType.RATIS,
THREE, new HashMap<>());
out.write(data.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
ReplicationConfig.fromTypeAndFactor(RATIS, THREE),
data.getBytes(UTF_8));
OzoneKey key = bucket.getKey(keyName);
assertEquals(keyName, key.getName());
try (OzoneInputStream is = bucket.readKey(keyName)) {
Expand Down Expand Up @@ -2185,11 +2174,9 @@ private void createAndCorruptKey(String volumeName, String bucketName,
OzoneBucket bucket = volume.getBucket(bucketName);

// Write data into a key
OzoneOutputStream out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, ReplicationType.RATIS,
ONE, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
ReplicationConfig.fromTypeAndFactor(RATIS, ONE),
value.getBytes(UTF_8));

// We need to find the location of the chunk file corresponding to the
// data we just wrote.
Expand Down Expand Up @@ -2247,11 +2234,9 @@ public void testGetKeyDetails() throws IOException {
String keyValue = RandomStringUtils.random(128);
//String keyValue = "this is a test value.glx";
// create the initial key with size 0, write will allocate the first block.
OzoneOutputStream out = bucket.createKey(keyName,
keyValue.getBytes(UTF_8).length, RATIS,
ONE, new HashMap<>());
out.write(keyValue.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
ReplicationConfig.fromTypeAndFactor(RATIS, ONE),
keyValue.getBytes(UTF_8));

// First, confirm the key info from the client matches the info in OM.
OmKeyArgs.Builder builder = new OmKeyArgs.Builder();
Expand Down Expand Up @@ -2342,11 +2327,9 @@ public void testReadKeyWithCorruptedData() throws IOException {
String keyName = UUID.randomUUID().toString();

// Write data into a key
OzoneOutputStream out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, ReplicationType.RATIS,
ONE, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
ReplicationConfig.fromTypeAndFactor(RATIS, ONE),
value.getBytes(UTF_8));

// We need to find the location of the chunk file corresponding to the
// data we just wrote.
Expand Down Expand Up @@ -2393,20 +2376,16 @@ void testZReadKeyWithUnhealthyContainerReplica() throws Exception {
String keyName1 = UUID.randomUUID().toString();

// Write first key
OzoneOutputStream out = bucket.createKey(keyName1,
value.getBytes(UTF_8).length, ReplicationType.RATIS,
THREE, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName1,
ReplicationConfig.fromTypeAndFactor(RATIS, THREE),
value.getBytes(UTF_8));

// Write second key
String keyName2 = UUID.randomUUID().toString();
value = "unhealthy container replica";
out = bucket.createKey(keyName2,
value.getBytes(UTF_8).length, ReplicationType.RATIS,
THREE, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName2,
ReplicationConfig.fromTypeAndFactor(RATIS, THREE),
value.getBytes(UTF_8));

// Find container ID
OzoneKey key = bucket.getKey(keyName2);
Expand Down Expand Up @@ -2491,11 +2470,9 @@ void testReadKeyWithCorruptedDataWithMutiNodes() throws IOException {
String keyName = UUID.randomUUID().toString();

// Write data into a key
OzoneOutputStream out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, ReplicationType.RATIS,
THREE, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
ReplicationConfig.fromTypeAndFactor(RATIS, THREE),
value.getBytes(UTF_8));

// We need to find the location of the chunk file corresponding to the
// data we just wrote.
Expand Down Expand Up @@ -2560,11 +2537,9 @@ public void testDeleteKey()
OzoneVolume volume = store.getVolume(volumeName);
volume.createBucket(bucketName);
OzoneBucket bucket = volume.getBucket(bucketName);
OzoneOutputStream out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, RATIS,
ONE, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
ReplicationConfig.fromTypeAndFactor(RATIS, ONE),
value.getBytes(UTF_8));
OzoneKey key = bucket.getKey(keyName);
assertEquals(keyName, key.getName());
bucket.deleteKey(keyName);
Expand Down Expand Up @@ -2853,30 +2828,18 @@ public void testListKey()
String keyBaseA = "key-a-";
for (int i = 0; i < 10; i++) {
byte[] value = RandomStringUtils.randomAscii(10240).getBytes(UTF_8);
OzoneOutputStream one = volAbucketA.createKey(
TestDataUtil.createKey(volAbucketA,
keyBaseA + i + "-" + RandomStringUtils.randomNumeric(5),
value.length, RATIS, ONE,
new HashMap<>());
one.write(value);
one.close();
OzoneOutputStream two = volAbucketB.createKey(
ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value);
TestDataUtil.createKey(volAbucketB,
keyBaseA + i + "-" + RandomStringUtils.randomNumeric(5),
value.length, RATIS, ONE,
new HashMap<>());
two.write(value);
two.close();
OzoneOutputStream three = volBbucketA.createKey(
ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value);
TestDataUtil.createKey(volBbucketA,
keyBaseA + i + "-" + RandomStringUtils.randomNumeric(5),
value.length, RATIS, ONE,
new HashMap<>());
three.write(value);
three.close();
OzoneOutputStream four = volBbucketB.createKey(
ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value);
TestDataUtil.createKey(volBbucketB,
keyBaseA + i + "-" + RandomStringUtils.randomNumeric(5),
value.length, RATIS, ONE,
new HashMap<>());
four.write(value);
four.close();
ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value);
}
/*
Create 10 keys in vol-a-<random>/buc-a-<random>,
Expand All @@ -2886,30 +2849,18 @@ public void testListKey()
String keyBaseB = "key-b-";
for (int i = 0; i < 10; i++) {
byte[] value = RandomStringUtils.randomAscii(10240).getBytes(UTF_8);
OzoneOutputStream one = volAbucketA.createKey(
TestDataUtil.createKey(volAbucketA,
keyBaseB + i + "-" + RandomStringUtils.randomNumeric(5),
value.length, RATIS, ONE,
new HashMap<>());
one.write(value);
one.close();
OzoneOutputStream two = volAbucketB.createKey(
ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value);
TestDataUtil.createKey(volAbucketB,
keyBaseB + i + "-" + RandomStringUtils.randomNumeric(5),
value.length, RATIS, ONE,
new HashMap<>());
two.write(value);
two.close();
OzoneOutputStream three = volBbucketA.createKey(
ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value);
TestDataUtil.createKey(volBbucketA,
keyBaseB + i + "-" + RandomStringUtils.randomNumeric(5),
value.length, RATIS, ONE,
new HashMap<>());
three.write(value);
three.close();
OzoneOutputStream four = volBbucketB.createKey(
ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value);
TestDataUtil.createKey(volBbucketB,
keyBaseB + i + "-" + RandomStringUtils.randomNumeric(5),
value.length, RATIS, ONE,
new HashMap<>());
four.write(value);
four.close();
ReplicationConfig.fromTypeAndFactor(RATIS, ONE), value);
}
Iterator<? extends OzoneKey> volABucketAIter =
volAbucketA.listKeys("key-");
Expand Down Expand Up @@ -3349,10 +3300,9 @@ public void testClientLeakDetector() throws Exception {
OzoneBucket bucket = volume.getBucket(bucketName);
byte[] data = new byte[10];
Arrays.fill(data, (byte) 1);
try (OzoneOutputStream out = bucket.createKey(keyName, 10,
ReplicationConfig.fromTypeAndFactor(RATIS, ONE), new HashMap<>())) {
out.write(data);
}
TestDataUtil.createKey(bucket, keyName,
ReplicationConfig.fromTypeAndFactor(RATIS, ONE),
data);
client = null;
System.gc();
GenericTestUtils.waitFor(() -> ozoneClientFactoryLogCapturer.getOutput()
Expand Down Expand Up @@ -4244,10 +4194,9 @@ private void validateOzoneAccessAcl(OzoneObj ozObj) throws IOException {
}

private void writeKey(String key1, OzoneBucket bucket) throws IOException {
OzoneOutputStream out = bucket.createKey(key1, 1024, RATIS,
ONE, new HashMap<>());
out.write(RandomStringUtils.random(1024).getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, key1,
ReplicationConfig.fromTypeAndFactor(RATIS, ONE),
RandomStringUtils.random(1024).getBytes(UTF_8));
}

private byte[] generateData(int size, byte val) {
Expand Down Expand Up @@ -4621,10 +4570,8 @@ public void testHeadObject() throws IOException {

String keyName = UUID.randomUUID().toString();

OzoneOutputStream out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, replicationConfig, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
replicationConfig, value.getBytes(UTF_8));

OzoneKey key = bucket.headObject(keyName);
assertEquals(volumeName, key.getVolumeName());
Expand Down Expand Up @@ -4666,16 +4613,12 @@ private void createRequiredForVersioningTest(String volumeName,
.setBucketLayout(BucketLayout.OBJECT_STORE).build());
OzoneBucket bucket = volume.getBucket(bucketName);

OzoneOutputStream out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, replicationConfig, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
replicationConfig, value.getBytes(UTF_8));

// Override key
out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, replicationConfig, new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
replicationConfig, value.getBytes(UTF_8));
}

private void checkExceptedResultForVersioningTest(String volumeName,
Expand Down Expand Up @@ -4896,11 +4839,9 @@ void testGetKeyAndFileWithNetworkTopology() throws IOException {
String keyName = UUID.randomUUID().toString();

// Write data into a key
try (OzoneOutputStream out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, ReplicationConfig.fromTypeAndFactor(
ReplicationType.RATIS, THREE), new HashMap<>())) {
out.write(value.getBytes(UTF_8));
}
TestDataUtil.createKey(bucket, keyName,
ReplicationConfig.fromTypeAndFactor(RATIS, THREE),
value.getBytes(UTF_8));

// Since the rpc client is outside of cluster, then getFirstNode should be
// equal to getClosestNode.
Expand Down Expand Up @@ -5181,10 +5122,8 @@ public void testPutObjectTagging(BucketLayout bucketLayout) throws Exception {

String keyName = UUID.randomUUID().toString();

OzoneOutputStream out = bucket.createKey(keyName,
value.getBytes(UTF_8).length, anyReplication(), new HashMap<>());
out.write(value.getBytes(UTF_8));
out.close();
TestDataUtil.createKey(bucket, keyName,
anyReplication(), value.getBytes(UTF_8));

OzoneKey key = bucket.getKey(keyName);
assertTrue(key.getTags().isEmpty());
Expand Down