diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 918cb3ec5635..59a5340826e3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1405,12 +1405,22 @@ public static boolean isKeyDeleted(String key, Table keyTable) { /** * Helper function for listStatus to find key in TableCache. + * If enableFileSystemPaths is false, create a fakeDir and add to cacheKeyMap + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKeyInTable Key from which listing needs to start. If startKey + * exists its status is included in the final list. + * @param keyName Key name of Key args. + * @param cacheKeyMap A map of keys to list. + * @param keyArgs A string converted from Key args. + * @param keyTable Key table from metadata manager. */ private void listStatusFindKeyInTableCache( - Iterator, CacheValue>> cacheIter, - String keyArgs, String startCacheKey, boolean recursive, - TreeMap cacheKeyMap) { - + boolean recursive, String startKeyInTable, String keyName, + TreeMap cacheKeyMap, String keyArgs, + Table keyTable) throws IOException { + Iterator, CacheValue>> + cacheIter = keyTable.cacheIterator(); while (cacheIter.hasNext()) { Map.Entry, CacheValue> entry = cacheIter.next(); @@ -1421,19 +1431,39 @@ private void listStatusFindKeyInTableCache( OmKeyInfo cacheOmKeyInfo = entry.getValue().getCacheValue(); // cacheOmKeyInfo is null if an entry is deleted in cache if (cacheOmKeyInfo != null - && cacheKey.startsWith(startCacheKey) - && cacheKey.compareTo(startCacheKey) >= 0) { + && cacheKey.startsWith(keyArgs) + && cacheKey.compareTo(startKeyInTable) >= 0) { + OzoneFileStatus fileStatus = new OzoneFileStatus( + cacheOmKeyInfo, scmBlockSize, !OzoneFSUtils.isFile(cacheKey)); if (!recursive) { - String remainingKey = StringUtils.stripEnd(cacheKey.substring( - startCacheKey.length()), OZONE_URI_DELIMITER); - // For non-recursive, the remaining part of key can't have '/' - if (remainingKey.contains(OZONE_URI_DELIMITER)) { - continue; + if (enableFileSystemPaths) { + String remainingKey = StringUtils.stripEnd(cacheKey.substring( + keyArgs.length()), OZONE_URI_DELIMITER); + // For non-recursive, the remaining part of key can't have '/' + if (remainingKey.contains(OZONE_URI_DELIMITER)) { + continue; + } + cacheKeyMap.putIfAbsent(cacheKey, fileStatus); + } else { + String entryKeyName = cacheOmKeyInfo.getKeyName(); + String immediateChild = OzoneFSUtils.getImmediateChild( + entryKeyName, keyName); + if (entryKeyName.equals(immediateChild)) { + cacheKeyMap.putIfAbsent(cacheKey, fileStatus); + } else { + //If immediateChild isn't equal entryKeyName, it is a fakeDir. + OmKeyInfo fakeDirEntry = createDirectoryKey( + cacheOmKeyInfo, immediateChild); + String fakeDirKey = metadataManager.getOzoneDirKey( + fakeDirEntry.getVolumeName(), fakeDirEntry.getBucketName(), + immediateChild); + cacheKeyMap.putIfAbsent(fakeDirKey, new OzoneFileStatus( + fakeDirEntry, scmBlockSize, true)); + } } + } else { + cacheKeyMap.putIfAbsent(cacheKey, fileStatus); } - OzoneFileStatus fileStatus = new OzoneFileStatus( - cacheOmKeyInfo, scmBlockSize, !OzoneFSUtils.isFile(cacheKey)); - cacheKeyMap.putIfAbsent(cacheKey, fileStatus); // This else block has been added to capture deleted entries in cache. // Adding deleted entries in cacheKeyMap as there is a possible race // condition where table cache iterator is flushed already when @@ -1489,14 +1519,15 @@ public List listStatus(OmKeyArgs args, boolean recursive, String startKey, long numEntries, String clientAddress, boolean allowPartialPrefixes) throws IOException { Preconditions.checkNotNull(args, "Key args can not be null"); - String volName = args.getVolumeName(); - String buckName = args.getBucketName(); + String volumeName = args.getVolumeName(); + String bucketName = args.getBucketName(); + String keyName = args.getKeyName(); List fileStatusList = new ArrayList<>(); if (numEntries <= 0) { return fileStatusList; } - if (isBucketFSOptimized(volName, buckName)) { + if (isBucketFSOptimized(volumeName, bucketName)) { Preconditions.checkArgument(!recursive); OzoneListStatusHelper statusHelper = new OzoneListStatusHelper(metadataManager, scmBlockSize, @@ -1508,9 +1539,6 @@ public List listStatus(OmKeyArgs args, boolean recursive, return buildFinalStatusList(statuses, args, clientAddress); } - String volumeName = args.getVolumeName(); - String bucketName = args.getBucketName(); - String keyName = args.getKeyName(); // A map sorted by OmKey to combine results from TableCache and DB. TreeMap cacheKeyMap = new TreeMap<>(); @@ -1527,25 +1555,20 @@ public List listStatus(OmKeyArgs args, boolean recursive, String keyArgs = OzoneFSUtils.addTrailingSlashIfNeeded( metadataManager.getOzoneKey(volumeName, bucketName, keyName)); - TableIterator> iterator; - Table keyTable; metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, bucketName); try { - keyTable = metadataManager - .getKeyTable(getBucketLayout(metadataManager, volName, buckName)); - iterator = getIteratorForKeyInTableCache(recursive, startKey, - volumeName, bucketName, cacheKeyMap, keyArgs, keyTable); + Table keyTable = metadataManager.getKeyTable( + getBucketLayout(metadataManager, volumeName, bucketName)); + String startKeyInTable = metadataManager.getOzoneKey( + volumeName, bucketName, startKey); + listStatusFindKeyInTableCache(recursive, startKeyInTable, keyName, + cacheKeyMap, keyArgs, keyTable); + listStatusFindKeyInDb(recursive, startKeyInTable, numEntries, + keyName, cacheKeyMap, keyArgs, keyTable); } finally { metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, - bucketName); - } - - try { - findKeyInDbWithIterator(recursive, startKey, numEntries, volumeName, - bucketName, keyName, cacheKeyMap, keyArgs, keyTable, iterator); - } finally { - iterator.close(); + bucketName); } int countEntries; @@ -1584,36 +1607,15 @@ public List listStatus(OmKeyArgs args, boolean recursive, return fileStatusList; } - private TableIterator> - getIteratorForKeyInTableCache( - boolean recursive, String startKey, String volumeName, String bucketName, - TreeMap cacheKeyMap, String keyArgs, - Table keyTable) throws IOException { - TableIterator> iterator; - Iterator, CacheValue>> - cacheIter = keyTable.cacheIterator(); - String startCacheKey = OZONE_URI_DELIMITER + volumeName + - OZONE_URI_DELIMITER + bucketName + OZONE_URI_DELIMITER + - ((startKey.equals(OZONE_URI_DELIMITER)) ? "" : startKey); - - // First, find key in TableCache - listStatusFindKeyInTableCache(cacheIter, keyArgs, startCacheKey, - recursive, cacheKeyMap); - iterator = keyTable.iterator(); - return iterator; - } - @SuppressWarnings("parameternumber") - private void findKeyInDbWithIterator(boolean recursive, String startKey, - long numEntries, String volumeName, String bucketName, String keyName, - TreeMap cacheKeyMap, String keyArgs, - Table keyTable, - TableIterator> iterator) + private void listStatusFindKeyInDb( + boolean recursive, String seekKeyInDb, long numEntries, String keyName, + TreeMap + cacheKeyMap, String keyArgs, Table keyTable) throws IOException { + TableIterator> iterator + = keyTable.iterator(); // Then, find key in DB - String seekKeyInDb = - metadataManager.getOzoneKey(volumeName, bucketName, startKey); Table.KeyValue entry = iterator.seek(seekKeyInDb); int countEntries = 0; if (iterator.hasNext()) { @@ -1626,6 +1628,8 @@ private void findKeyInDbWithIterator(boolean recursive, String startKey, entry = iterator.next(); String entryInDb = entry.getKey(); OmKeyInfo omKeyInfo = entry.getValue(); + String volumeName = omKeyInfo.getVolumeName(); + String bucketName = omKeyInfo.getBucketName(); if (entryInDb.startsWith(keyArgs)) { String entryKeyName = omKeyInfo.getKeyName(); if (recursive) { @@ -1655,7 +1659,9 @@ private void findKeyInDbWithIterator(boolean recursive, String startKey, if (!entryKeyName.equals(immediateChild)) { OmKeyInfo fakeDirEntry = createDirectoryKey( omKeyInfo, immediateChild); - cacheKeyMap.put(entryInDb, + String fakeDirKey = metadataManager.getOzoneDirKey( + volumeName, bucketName, immediateChild); + cacheKeyMap.put(fakeDirKey, new OzoneFileStatus(fakeDirEntry, scmBlockSize, true)); } else { @@ -1675,6 +1681,7 @@ private void findKeyInDbWithIterator(boolean recursive, String startKey, } } } + iterator.close(); } private List buildFinalStatusList(