From 2050ebb0668d5283733b34608333f9628b448b9a Mon Sep 17 00:00:00 2001 From: Duong Nguyen Date: Wed, 26 Oct 2022 18:06:12 -0700 Subject: [PATCH 1/3] HDDS-7419. Integrate the GetKeyInfo API to OFS --- .../hadoop/ozone/client/rpc/RpcClient.java | 12 +++++++- .../hadoop/ozone/om/helpers/OmKeyInfo.java | 30 +++++++++++++++++-- .../om/protocol/OzoneManagerProtocol.java | 2 ++ .../src/main/proto/OmClientProtocol.proto | 1 + .../hadoop/ozone/om/KeyManagerImpl.java | 23 ++++++++++++-- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 1 + .../BasicRootedOzoneClientAdapterImpl.java | 1 + 7 files changed, 65 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 1f970ecac61c..0bbe7d5dbdff 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1698,7 +1698,17 @@ public OzoneInputStream readFile(String volumeName, String bucketName, .setSortDatanodesInPipeline(topologyAwareReadEnabled) .setLatestVersionLocation(getLatestVersionLocation) .build(); - OmKeyInfo keyInfo = ozoneManagerClient.lookupFile(keyArgs); + final OmKeyInfo keyInfo; + if (omVersion.compareTo(OzoneManagerVersion.OPTIMIZED_GET_KEY_INFO) >= 0) { + keyInfo = ozoneManagerClient.getKeyInfo(keyArgs, false) + .getKeyInfo(); + if (!keyInfo.isFile()) { + throw new OMException(keyName + " is not a file.", + OMException.ResultCodes.NOT_A_FILE); + } + } else { + keyInfo = ozoneManagerClient.lookupFile(keyArgs); + } return getInputStreamWithRetryFunction(keyInfo); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index f8f589af2efc..209af9f7b25c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -61,6 +61,10 @@ public final class OmKeyInfo extends WithParentObjectId { private ReplicationConfig replicationConfig; private FileEncryptionInfo encInfo; private FileChecksum fileChecksum; + /** + * Support OSF use-case to identify if the key is a file or a directory. + */ + private boolean isFile; /** * Represents leaf node name. This also will be used when the keyName is @@ -106,12 +110,13 @@ public final class OmKeyInfo extends WithParentObjectId { Map metadata, FileEncryptionInfo encInfo, List acls, long parentObjectID, long objectID, long updateID, - FileChecksum fileChecksum) { + FileChecksum fileChecksum, boolean isFile) { this(volumeName, bucketName, keyName, versions, dataSize, creationTime, modificationTime, replicationConfig, metadata, encInfo, acls, objectID, updateID, fileChecksum); this.fileName = fileName; this.parentObjectID = parentObjectID; + this.isFile = isFile; } public String getVolumeName() { @@ -177,6 +182,14 @@ public void updateModifcationTime() { this.modificationTime = Time.monotonicNow(); } + public void setFile(boolean file) { + isFile = file; + } + + public boolean isFile() { + return isFile; + } + /** * updates the length of the each block in the list given. * This will be called when the key is being committed to OzoneManager. @@ -409,6 +422,8 @@ public static class Builder { private long parentObjectID; private FileChecksum fileChecksum; + private boolean isFile; + public Builder() { this.metadata = new HashMap<>(); omKeyLocationInfoGroups = new ArrayList<>(); @@ -520,12 +535,17 @@ public Builder setFileChecksum(FileChecksum checksum) { return this; } + public Builder setFile(boolean isAFile) { + this.isFile = isAFile; + return this; + } + public OmKeyInfo build() { return new OmKeyInfo( volumeName, bucketName, keyName, fileName, omKeyLocationInfoGroups, dataSize, creationTime, modificationTime, replicationConfig, metadata, encInfo, acls, - parentObjectID, objectID, updateID, fileChecksum); + parentObjectID, objectID, updateID, fileChecksum, isFile); } } @@ -627,6 +647,7 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName, if (encInfo != null) { kb.setFileEncryptionInfo(OMPBHelper.convert(encInfo)); } + kb.setIsFile(isFile); return kb.build(); } @@ -669,6 +690,11 @@ public static OmKeyInfo getFromProtobuf(KeyInfo keyInfo) throws IOException { FileChecksum fileChecksum = OMPBHelper.convert(keyInfo.getFileChecksum()); builder.setFileChecksum(fileChecksum); } + + if (keyInfo.hasIsFile()) { + builder.setFile(keyInfo.getIsFile()); + } + // not persisted to DB. FileName will be filtered out from keyName builder.setFileName(OzoneFSUtils.getFileName(keyInfo.getKeyName())); return builder.build(); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 2bd30eee721c..d9a758127536 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -759,7 +759,9 @@ default OpenKeySession createFile(OmKeyArgs keyArgs, boolean overWrite, * if bucket does not exist * @throws IOException if there is error in the db * invalid arguments + * @deprecated use {@link OzoneManagerProtocol#getKeyInfo} instead. */ + @Deprecated OmKeyInfo lookupFile(OmKeyArgs keyArgs) throws IOException; /** diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 76f224b7d7ad..5ee314643576 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -953,6 +953,7 @@ message KeyInfo { optional uint64 parentID = 16; optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 17; optional FileChecksumProto fileChecksum = 18; + optional bool isFile = 19; } message DirectoryInfo { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index dd66c33a1e3b..2aec21c6ce27 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -367,7 +367,7 @@ private OmKeyInfo readKeyInfo(OmKeyArgs args) throws IOException { if (bucketLayout.isFileSystemOptimized()) { value = getOmKeyInfoFSO(volumeName, bucketName, keyName); } else { - value = getOmKeyInfo(volumeName, bucketName, keyName); + value = getOmKeyInfoDirectoryAware(volumeName, bucketName, keyName); } } catch (IOException ex) { if (ex instanceof OMException) { @@ -396,8 +396,26 @@ private OmKeyInfo readKeyInfo(OmKeyArgs args) throws IOException { return value; } + private OmKeyInfo getOmKeyInfoDirectoryAware(String volumeName, + String bucketName, String keyName) throws IOException { + OmKeyInfo keyInfo = getOmKeyInfo(volumeName, bucketName, keyName); + + // Check if the key is a directory. + if (keyInfo != null) { + keyInfo.setFile(true); + return keyInfo; + } + + String dirKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); + OmKeyInfo dirKeyInfo = getOmKeyInfo(volumeName, bucketName, dirKey); + if (dirKeyInfo != null) { + dirKeyInfo.setFile(false); + } + return dirKeyInfo; + } + private OmKeyInfo getOmKeyInfo(String volumeName, String bucketName, - String keyName) throws IOException { + String keyName) throws IOException { String keyBytes = metadataManager.getOzoneKey(volumeName, bucketName, keyName); BucketLayout bucketLayout = getBucketLayout(metadataManager, volumeName, @@ -425,6 +443,7 @@ private OmKeyInfo getOmKeyInfoFSO(String volumeName, String bucketName, fileStatus.getKeyInfo().getKeyName()); fileStatus.getKeyInfo().setKeyName(keyPath); } + fileStatus.getKeyInfo().setFile(fileStatus.isFile()); return fileStatus.getKeyInfo(); } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index 25e4ffbb1874..f7a7326296d0 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@ -219,6 +219,7 @@ public InputStream readFile(String key) throws IOException { return bucket.readFile(key).getInputStream(); } catch (OMException ex) { if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND + || ex.getResult() == OMException.ResultCodes.KEY_NOT_FOUND || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) { throw new FileNotFoundException( ex.getResult().name() + ": " + ex.getMessage()); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 724a76bb786a..d04206213806 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -351,6 +351,7 @@ public InputStream readFile(String pathStr) throws IOException { return bucket.readFile(key).getInputStream(); } catch (OMException ex) { if (ex.getResult() == OMException.ResultCodes.FILE_NOT_FOUND + || ex.getResult() == OMException.ResultCodes.KEY_NOT_FOUND || ex.getResult() == OMException.ResultCodes.NOT_A_FILE) { throw new FileNotFoundException( ex.getResult().name() + ": " + ex.getMessage()); From 42aed99b8ba658d58ed3b1be851426f0c9989c4e Mon Sep 17 00:00:00 2001 From: Duong Nguyen Date: Wed, 26 Oct 2022 20:55:29 -0700 Subject: [PATCH 2/3] Update integration test. --- .../org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index f3e8cf10bee3..58ffd4256e07 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -1005,10 +1005,10 @@ public void testSeekOnFileLength() throws IOException { Path fileNotExists = new Path("/file_notexist"); try { fs.open(fileNotExists); - Assert.fail("Should throw FILE_NOT_FOUND error as file doesn't exist!"); + Assert.fail("Should throw FileNotFoundException as file doesn't exist!"); } catch (FileNotFoundException fnfe) { - Assert.assertTrue("Expected FILE_NOT_FOUND error", - fnfe.getMessage().contains("FILE_NOT_FOUND")); + Assert.assertTrue("Expected KEY_NOT_FOUND error", + fnfe.getMessage().contains("KEY_NOT_FOUND")); } } From c7c73705022aa9b1a03aae2b3e48e1372500c511 Mon Sep 17 00:00:00 2001 From: Duong Nguyen Date: Mon, 31 Oct 2022 23:11:55 -0700 Subject: [PATCH 3/3] Correct javadoc. --- .../main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index 209af9f7b25c..62b06c7c0d72 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -62,7 +62,7 @@ public final class OmKeyInfo extends WithParentObjectId { private FileEncryptionInfo encInfo; private FileChecksum fileChecksum; /** - * Support OSF use-case to identify if the key is a file or a directory. + * Support OFS use-case to identify if the key is a file or a directory. */ private boolean isFile;