diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 3f984d279161..d72d930e54f3 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -1030,6 +1030,7 @@ public OzoneFileStatus getOzoneFileStatus(String volumeName, .setBucketName(bucketName) .setKeyName(keyName) .setRefreshPipeline(true) + .setSortDatanodesInPipeline(topologyAwareReadEnabled) .build(); return ozoneManagerClient.getFileStatus(keyArgs); } @@ -1112,6 +1113,7 @@ public List listStatus(String volumeName, String bucketName, .setBucketName(bucketName) .setKeyName(keyName) .setRefreshPipeline(true) + .setSortDatanodesInPipeline(topologyAwareReadEnabled) .build(); return ozoneManagerClient .listStatus(keyArgs, recursive, startKey, numEntries); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index a6ea0423bcf0..506d84cbca19 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -1177,6 +1177,7 @@ public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException { .setVolumeName(args.getVolumeName()) .setBucketName(args.getBucketName()) .setKeyName(args.getKeyName()) + .setSortDatanodes(args.getSortDatanodes()) .build(); GetFileStatusRequest req = GetFileStatusRequest.newBuilder() @@ -1390,6 +1391,7 @@ public List listStatus(OmKeyArgs args, boolean recursive, .setVolumeName(args.getVolumeName()) .setBucketName(args.getBucketName()) .setKeyName(args.getKeyName()) + .setSortDatanodes(args.getSortDatanodes()) .build(); ListStatusRequest listStatusRequest = ListStatusRequest.newBuilder() diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index a2b70a52f2f7..e4fe449d3ab2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -1095,8 +1095,7 @@ public void testListStatus() throws IOException { tempFileStatus = keyManager.listStatus(dirArgs, false, tempFileStatus != null ? tempFileStatus.get(tempFileStatus.size() - 1).getKeyInfo() - .getKeyName() : null, - 2); + .getKeyName() : null, 2); tmpStatusSet.addAll(tempFileStatus); } while (tempFileStatus.size() == 2); verifyFileStatus(directory, new ArrayList<>(tmpStatusSet), directorySet, @@ -1112,9 +1111,7 @@ public void testListStatus() throws IOException { tempFileStatus = keyManager.listStatus(dirArgs, true, tempFileStatus != null ? tempFileStatus.get(tempFileStatus.size() - 1).getKeyInfo() - .getKeyName() : - null, - 2); + .getKeyName() : null, 2); tmpStatusSet.addAll(tempFileStatus); } while (tempFileStatus.size() == 2); verifyFileStatus(directory, new ArrayList<>(tmpStatusSet), directorySet, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 28e091df497f..14fc07e34864 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1732,17 +1732,36 @@ private void validateOzoneObj(OzoneObj obj) throws OMException { * @param args Key args * @throws OMException if file does not exist * if bucket does not exist + * if volume does not exist * @throws IOException if there is error in the db * invalid arguments */ public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException { Preconditions.checkNotNull(args, "Key args can not be null"); + return getFileStatus(args, null); + } + + /** + * OzoneFS api to get file status for an entry. + * + * @param args Key args + * @param clientAddress a hint to key manager, order the datanode in returned + * pipeline by distance between client and datanode. + * @throws OMException if file does not exist + * if bucket does not exist + * if volume does not exist + * @throws IOException if there is error in the db + * invalid arguments + */ + public OzoneFileStatus getFileStatus(OmKeyArgs args, String clientAddress) + throws IOException { + Preconditions.checkNotNull(args, "Key args can not be null"); String volumeName = args.getVolumeName(); String bucketName = args.getBucketName(); String keyName = args.getKeyName(); return getOzoneFileStatus(volumeName, bucketName, keyName, - args.getRefreshPipeline(), false, null); + args.getRefreshPipeline(), args.getSortDatanodes(), clientAddress); } private OzoneFileStatus getOzoneFileStatus(String volumeName, @@ -1783,6 +1802,9 @@ private OzoneFileStatus getOzoneFileStatus(String volumeName, // if the key is a file then do refresh pipeline info in OM by asking SCM if (fileKeyInfo != null) { + // refreshPipeline flag check has been removed as part of + // https://issues.apache.org/jira/browse/HDDS-3658. + // Please refer this jira for more details. refreshPipeline(fileKeyInfo); if (sortDatanodes) { sortDatanodeInPipeline(fileKeyInfo, clientAddress); @@ -2011,7 +2033,27 @@ private void listStatusFindKeyInTableCache( * @return list of file status */ public List listStatus(OmKeyArgs args, boolean recursive, - String startKey, long numEntries) throws IOException { + String startKey, long numEntries) + throws IOException { + return listStatus(args, recursive, startKey, numEntries, null); + } + + /** + * List the status for a file or a directory and its contents. + * + * @param args Key args + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey exists + * its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @param clientAddress a hint to key manager, order the datanode in returned + * pipeline by distance between client and datanode. + * @return list of file status + */ + public List listStatus(OmKeyArgs args, boolean recursive, + String startKey, long numEntries, String clientAddress) + throws IOException { Preconditions.checkNotNull(args, "Key args can not be null"); List fileStatusList = new ArrayList<>(); @@ -2027,18 +2069,18 @@ public List listStatus(OmKeyArgs args, boolean recursive, // A set to keep track of keys deleted in cache but not flushed to DB. Set deletedKeySet = new TreeSet<>(); + if (Strings.isNullOrEmpty(startKey)) { + OzoneFileStatus fileStatus = getFileStatus(args, clientAddress); + if (fileStatus.isFile()) { + return Collections.singletonList(fileStatus); + } + // keyName is a directory + startKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); + } + metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, bucketName); try { - if (Strings.isNullOrEmpty(startKey)) { - OzoneFileStatus fileStatus = getFileStatus(args); - if (fileStatus.isFile()) { - return Collections.singletonList(fileStatus); - } - // keyName is a directory - startKey = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); - } - Table keyTable = metadataManager.getKeyTable(); Iterator, CacheValue>> cacheIter = keyTable.cacheIterator(); @@ -2144,10 +2186,14 @@ public List listStatus(OmKeyArgs args, boolean recursive, metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, bucketName); } - if (args.getRefreshPipeline()) { - for(OzoneFileStatus fileStatus : fileStatusList){ + + for (OzoneFileStatus fileStatus : fileStatusList) { + if (args.getRefreshPipeline()) { refreshPipeline(fileStatus.getKeyInfo()); } + if (args.getSortDatanodes()) { + sortDatanodeInPipeline(fileStatus.getKeyInfo(), clientAddress); + } } return fileStatusList; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 9daeec16fe46..e43524aee2a7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2797,7 +2797,7 @@ public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException { try { metrics.incNumGetFileStatus(); - return keyManager.getFileStatus(args); + return keyManager.getFileStatus(args, getClientAddress()); } catch (IOException ex) { metrics.incNumGetFileStatusFails(); auditSuccess = false; @@ -2921,7 +2921,8 @@ public List listStatus(OmKeyArgs args, boolean recursive, try { metrics.incNumListStatus(); - return keyManager.listStatus(args, recursive, startKey, numEntries); + return keyManager.listStatus(args, recursive, startKey, numEntries, + getClientAddress()); } catch (Exception ex) { metrics.incNumListStatusFails(); auditSuccess = false; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/fs/OzoneManagerFS.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/fs/OzoneManagerFS.java index 647931af0d04..a7329c81ca3b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/fs/OzoneManagerFS.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/fs/OzoneManagerFS.java @@ -31,8 +31,28 @@ * Ozone Manager FileSystem interface. */ public interface OzoneManagerFS extends IOzoneAcl { + + /** + * Get file status for a file or a directory. + * + * @param args the args of the key provided by client. + * @return file status. + * @throws IOException if file or bucket or volume does not exist + */ OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException; + /** + * Get file status for a file or a directory. + * + * @param args the args of the key provided by client. + * @param clientAddress a hint to key manager, order the datanode in returned + * pipeline by distance between client and datanode. + * @return file status. + * @throws IOException if file or bucket or volume does not exist + */ + OzoneFileStatus getFileStatus(OmKeyArgs args, String clientAddress) + throws IOException; + void createDirectory(OmKeyArgs args) throws IOException; OpenKeySession createFile(OmKeyArgs args, boolean isOverWrite, @@ -49,6 +69,37 @@ OpenKeySession createFile(OmKeyArgs args, boolean isOverWrite, */ OmKeyInfo lookupFile(OmKeyArgs args, String clientAddress) throws IOException; + /** + * List the status for a file or a directory and its contents. + * + * @param keyArgs the args of the key provided by client. + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey + * exists its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @return list of file status + * @throws IOException if file or bucket or volume does not exist + */ + List listStatus(OmKeyArgs keyArgs, boolean recursive, + String startKey, long numEntries) + throws IOException; + + /** + * List the status for a file or a directory and its contents. + * + * @param keyArgs the args of the key provided by client. + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey + * exists its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @param clientAddress a hint to key manager, order the datanode in returned + * pipeline by distance between client and datanode. + * @return list of file status + * @throws IOException if file or bucket or volume does not exist + */ List listStatus(OmKeyArgs keyArgs, boolean recursive, - String startKey, long numEntries) throws IOException; + String startKey, long numEntries, String clientAddress) + throws IOException; }