Skip to content

HDDS-9072. Set isFile when creating OmKeyInfo for createFile code path #5108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
smengcl merged 2 commits into from
Jul 26, 2023
Merged

HDDS-9072. Set isFile when creating OmKeyInfo for createFile code path #5108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
85a85c1
Set isFile when creating OmKeyInfo for createFile code path
hemantk-12 Jul 24, 2023
8b62bc7
Addressed code review comments
hemantk-12 Jul 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 43 additions & 29 deletions ...p-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.VolumeArgs;
Expand All @@ -40,7 +41,6 @@
import org.apache.hadoop.ozone.om.ha.HadoopRpcOMFailoverProxyProvider;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServerConfig;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
Expand All @@ -49,6 +49,7 @@
import java.io.IOException;
import java.net.ConnectException;
import java.time.Duration;
import java.util.Iterator;
import java.util.UUID;
import java.util.HashMap;

Expand All @@ -60,9 +61,13 @@
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;

import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK;
import static org.junit.Assert.fail;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assertions.assertEquals;

/**
* Base class for Ozone Manager HA tests.
Expand Down Expand Up @@ -245,17 +250,17 @@ protected OzoneBucket setupBucket() throws Exception {
objectStore.createVolume(volumeName, createVolumeArgs);
OzoneVolume retVolumeinfo = objectStore.getVolume(volumeName);

Assert.assertTrue(retVolumeinfo.getName().equals(volumeName));
Assert.assertTrue(retVolumeinfo.getOwner().equals(userName));
Assert.assertTrue(retVolumeinfo.getAdmin().equals(adminName));
assertEquals(volumeName, retVolumeinfo.getName());
assertEquals(userName, retVolumeinfo.getOwner());
assertEquals(adminName, retVolumeinfo.getAdmin());

String bucketName = UUID.randomUUID().toString();
retVolumeinfo.createBucket(bucketName);

OzoneBucket ozoneBucket = retVolumeinfo.getBucket(bucketName);

Assert.assertTrue(ozoneBucket.getName().equals(bucketName));
Assert.assertTrue(ozoneBucket.getVolumeName().equals(volumeName));
assertEquals(bucketName, ozoneBucket.getName());
assertEquals(volumeName, ozoneBucket.getVolumeName());

return ozoneBucket;
}
Expand All @@ -278,26 +283,24 @@ protected OzoneBucket linkBucket(OzoneBucket srcBuk) throws Exception {
objectStore.createVolume(linkedVolName, createVolumeArgs);
OzoneVolume linkedVolumeInfo = objectStore.getVolume(linkedVolName);

Assert.assertTrue(linkedVolumeInfo.getName().equals(linkedVolName));
Assert.assertTrue(linkedVolumeInfo.getOwner().equals(userName));
Assert.assertTrue(linkedVolumeInfo.getAdmin().equals(adminName));
assertEquals(linkedVolName, linkedVolumeInfo.getName());
assertEquals(userName, linkedVolumeInfo.getOwner());
assertEquals(adminName, linkedVolumeInfo.getAdmin());

String linkedBucketName = UUID.randomUUID().toString();
linkedVolumeInfo.createBucket(linkedBucketName, createBucketArgs);

OzoneBucket linkedBucket = linkedVolumeInfo.getBucket(linkedBucketName);

Assert.assertTrue(linkedBucket.getName().equals(linkedBucketName));
Assert.assertTrue(linkedBucket.getVolumeName().equals(linkedVolName));
Assert.assertTrue(linkedBucket.isLink());
assertEquals(linkedBucketName, linkedBucket.getName());
assertEquals(linkedVolName, linkedBucket.getVolumeName());
assertTrue(linkedBucket.isLink());

return linkedBucket;
}

/**
* Stop the current leader OM.
*
* @throws Exception
*/
protected void stopLeaderOM() {
//Stop the leader OM.
Expand Down Expand Up @@ -332,9 +335,9 @@ protected void createVolumeTest(boolean checkSuccess) throws Exception {
OzoneVolume retVolumeinfo = objectStore.getVolume(volumeName);

if (checkSuccess) {
Assert.assertTrue(retVolumeinfo.getName().equals(volumeName));
Assert.assertTrue(retVolumeinfo.getOwner().equals(userName));
Assert.assertTrue(retVolumeinfo.getAdmin().equals(adminName));
assertEquals(volumeName, retVolumeinfo.getName());
assertEquals(userName, retVolumeinfo.getOwner());
assertEquals(adminName, retVolumeinfo.getAdmin());
} else {
// Verify that the request failed
fail("There is no quorum. Request should have failed");
Expand Down Expand Up @@ -383,16 +386,27 @@ protected void testCreateFile(OzoneBucket ozoneBucket, String keyName,

OzoneKeyDetails ozoneKeyDetails = ozoneBucket.getKey(keyName);

Assert.assertEquals(keyName, ozoneKeyDetails.getName());
Assert.assertEquals(ozoneBucket.getName(), ozoneKeyDetails.getBucketName());
Assert.assertEquals(ozoneBucket.getVolumeName(),
assertEquals(keyName, ozoneKeyDetails.getName());
assertEquals(ozoneBucket.getName(), ozoneKeyDetails.getBucketName());
assertEquals(ozoneBucket.getVolumeName(),
ozoneKeyDetails.getVolumeName());
Assert.assertEquals(data.length(), ozoneKeyDetails.getDataSize());
assertEquals(data.length(), ozoneKeyDetails.getDataSize());
assertTrue(ozoneKeyDetails.isFile());

try (OzoneInputStream ozoneInputStream = ozoneBucket.readKey(keyName)) {
byte[] fileContent = new byte[data.getBytes(UTF_8).length];
ozoneInputStream.read(fileContent);
Assert.assertEquals(data, new String(fileContent, UTF_8));
assertEquals(data, new String(fileContent, UTF_8));
}

Iterator<? extends OzoneKey> iterator = ozoneBucket.listKeys("/");
while (iterator.hasNext()) {
OzoneKey ozoneKey = iterator.next();
if (!ozoneKey.getName().endsWith(OM_KEY_PREFIX)) {
assertTrue(ozoneKey.isFile());
} else {
assertFalse(ozoneKey.isFile());
}
}
}

Expand All @@ -411,18 +425,18 @@ protected void createKeyTest(boolean checkSuccess) throws Exception {

OzoneVolume retVolumeinfo = getObjectStore().getVolume(volumeName);

Assert.assertTrue(retVolumeinfo.getName().equals(volumeName));
Assert.assertTrue(retVolumeinfo.getOwner().equals(userName));
Assert.assertTrue(retVolumeinfo.getAdmin().equals(adminName));
assertEquals(volumeName, retVolumeinfo.getName());
assertEquals(userName, retVolumeinfo.getOwner());
assertEquals(adminName, retVolumeinfo.getAdmin());

String bucketName = UUID.randomUUID().toString();
String keyName = UUID.randomUUID().toString();
retVolumeinfo.createBucket(bucketName);

OzoneBucket ozoneBucket = retVolumeinfo.getBucket(bucketName);

Assert.assertTrue(ozoneBucket.getName().equals(bucketName));
Assert.assertTrue(ozoneBucket.getVolumeName().equals(volumeName));
assertEquals(bucketName, ozoneBucket.getName());
assertEquals(volumeName, ozoneBucket.getVolumeName());

String value = "random data";
OzoneOutputStream ozoneOutputStream = ozoneBucket.createKey(keyName,
Expand All @@ -434,7 +448,7 @@ protected void createKeyTest(boolean checkSuccess) throws Exception {
try (OzoneInputStream ozoneInputStream = ozoneBucket.readKey(keyName)) {
byte[] fileContent = new byte[value.getBytes(UTF_8).length];
ozoneInputStream.read(fileContent);
Assert.assertEquals(value, new String(fileContent, UTF_8));
assertEquals(value, new String(fileContent, UTF_8));
}

} catch (IOException e) {
Expand Down
33 changes: 18 additions & 15 deletions ...zone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -686,30 +686,33 @@ protected OmKeyInfo prepareFileInfo(
* @return OmKeyInfo
*/
@SuppressWarnings("parameterNumber")
protected OmKeyInfo createFileInfo(@Nonnull KeyArgs keyArgs,
protected OmKeyInfo createFileInfo(
@Nonnull KeyArgs keyArgs,
@Nonnull List<OmKeyLocationInfo> locations,
@Nonnull ReplicationConfig replicationConfig,
long size,
@Nullable FileEncryptionInfo encInfo,
@Nonnull PrefixManager prefixManager,
@Nullable OmBucketInfo omBucketInfo,
OMFileRequest.OMPathInfoWithFSO omPathInfo,
long transactionLogIndex, long objectID) {
long transactionLogIndex, long objectID
) {
OmKeyInfo.Builder builder = new OmKeyInfo.Builder();
builder.setVolumeName(keyArgs.getVolumeName())
.setBucketName(keyArgs.getBucketName())
.setKeyName(keyArgs.getKeyName())
.setOmKeyLocationInfos(Collections.singletonList(
new OmKeyLocationInfoGroup(0, locations)))
.setCreationTime(keyArgs.getModificationTime())
.setModificationTime(keyArgs.getModificationTime())
.setDataSize(size)
.setReplicationConfig(replicationConfig)
.setFileEncryptionInfo(encInfo)
.setAcls(getAclsForKey(keyArgs, omBucketInfo, prefixManager))
.addAllMetadata(KeyValueUtil.getFromProtobuf(
keyArgs.getMetadataList()))
.setUpdateID(transactionLogIndex);
.setBucketName(keyArgs.getBucketName())
.setKeyName(keyArgs.getKeyName())
.setOmKeyLocationInfos(Collections.singletonList(
new OmKeyLocationInfoGroup(0, locations)))
.setCreationTime(keyArgs.getModificationTime())
.setModificationTime(keyArgs.getModificationTime())
.setDataSize(size)
.setReplicationConfig(replicationConfig)
.setFileEncryptionInfo(encInfo)
.setAcls(getAclsForKey(keyArgs, omBucketInfo, prefixManager))
.addAllMetadata(KeyValueUtil.getFromProtobuf(
keyArgs.getMetadataList()))
.setUpdateID(transactionLogIndex)
.setFile(true);
if (omPathInfo != null) {
// FileTable metadata format
objectID = omPathInfo.getLeafNodeObjectId();
Expand Down