From c6ba8f76440be283685fb9f507f684f6c21c350b Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Mon, 23 May 2022 07:31:26 +0530 Subject: [PATCH 01/13] HDDS-6788. Sort Ozone list Status output for FSO buckets. --- .../hadoop/ozone/om/TestListStatus.java | 198 ++++++++ .../hadoop/ozone/om/KeyManagerImpl.java | 14 +- .../ozone/om/OzoneListStatusHelper.java | 440 ++++++++++++++++++ 3 files changed, 650 insertions(+), 2 deletions(-) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java new file mode 100644 index 000000000000..982cd117281a --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om; + +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.TestDataUtil; +import org.apache.hadoop.ozone.client.*; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.junit.*; +import org.junit.rules.Timeout; + +import java.io.IOException; +import java.util.*; + +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; + +/** + * Check that list status output is sorted. + */ +public class TestListStatus { + + private static MiniOzoneCluster cluster = null; + private static OzoneConfiguration conf; + private static String clusterId; + private static String scmId; + private static String omId; + + private static OzoneBucket legacyOzoneBucket; + private static OzoneBucket fsoOzoneBucket; + + @Rule + public Timeout timeout = new Timeout(1200000); + + /** + * Create a MiniDFSCluster for testing. + *

+ * + * @throws IOException + */ + @BeforeClass + public static void init() throws Exception { + conf = new OzoneConfiguration(); + conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, + true); + clusterId = UUID.randomUUID().toString(); + scmId = UUID.randomUUID().toString(); + omId = UUID.randomUUID().toString(); + cluster = MiniOzoneCluster.newBuilder(conf).setClusterId(clusterId) + .setScmId(scmId).setOmId(omId).build(); + cluster.waitForClusterToBeReady(); + + // create a volume and a LEGACY bucket + legacyOzoneBucket = TestDataUtil + .createVolumeAndBucket(cluster, BucketLayout.LEGACY); + String volumeName = legacyOzoneBucket.getVolumeName(); + + // create a volume and a FSO bucket + BucketArgs omBucketArgs; + BucketArgs.Builder builder = BucketArgs.newBuilder(); + builder.setStorageType(StorageType.DISK); + builder.setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED); + omBucketArgs = builder.build(); + OzoneClient client = cluster.getClient(); + OzoneVolume ozoneVolume = client.getObjectStore().getVolume(volumeName); + + String fsoBucketName = "bucket" + RandomStringUtils.randomNumeric(5); + ozoneVolume.createBucket(fsoBucketName, omBucketArgs); + fsoOzoneBucket = ozoneVolume.getBucket(fsoBucketName); + + // Set the number of keys to be processed during batch operate. + conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5); + + initFSNameSpace(); + } + + @AfterClass + public static void teardownClass() { + if (cluster != null) { + cluster.shutdown(); + } + } + + private static void initFSNameSpace() throws Exception { + /* + Keys Namespace: + "a1" Dir + "a1/a11" Dir + "a1/a12" File + "a1/a13" File + "a2" File + "a3" Dir + "a3/a31" Dir + "a3/a32" File + "a8" File + "a9" Dir + "a10" File + + "b1" File + "b2" File + "b3" File + "b4" File + */ + buildNameSpaceTree(fsoOzoneBucket); + } + + @Test + public void testSortedListStatus() throws Exception { + // a) test if output is sorted + checkKeyList("", "", 1000, 10); + + // b) number of keys returns is expected + checkKeyList("", "", 2, 2); + + // c) check if full prefix works + checkKeyList("a1", "", 100, 3); + + // d) check if full prefix with numEntries work + checkKeyList("a1", "", 2, 2); + + // e) check if existing start key >>> + checkKeyList("a1", "a1/a12", 100, 2); + + // f) check with non existing start key>>> + checkKeyList("", "a7", 100, 6); + + // g) check if half prefix works <<<< + // checkKeyList("b", "", 100, 4); + + // h) check half prefix with non-existing start key + // checkKeyList("b", "b5", 100, 2); + } + + private static void createFile(OzoneBucket bucket, String keyName) throws IOException { + try (OzoneOutputStream oos = bucket.createFile(keyName, 0, + RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), + true, false)) { + + } + } + private static void buildNameSpaceTree(OzoneBucket ozoneBucket) + throws Exception { + ozoneBucket.createDirectory("/a1"); + createFile(ozoneBucket, "/a2"); + ozoneBucket.createDirectory("/a3"); + createFile(ozoneBucket, "/a8"); + ozoneBucket.createDirectory("/a9"); + createFile(ozoneBucket, "/a10"); + + ozoneBucket.createDirectory("/a1/a11"); + createFile(ozoneBucket,"/a1/a12"); + createFile(ozoneBucket,"/a1/a13"); + + ozoneBucket.createDirectory("/a3/a31"); + createFile(ozoneBucket,"/a3/a32"); + + createFile(ozoneBucket,"/b1"); + createFile(ozoneBucket,"/b2"); + createFile(ozoneBucket,"/b7"); + createFile(ozoneBucket,"/b8"); + } + + private void checkKeyList(String keyPrefix, String startKey, + long numEntries, int expectedNumKeys) throws Exception { + + List statuses = + fsoOzoneBucket.listStatus(keyPrefix, false, startKey, numEntries); + Assert.assertEquals(expectedNumKeys, statuses.size()); + + System.out.println("BEGIN:::keyPrefix---> " + keyPrefix + ":::---> " + + startKey); + for (OzoneFileStatus st : statuses) { + System.out.println("status:" + st); + } + System.out.println("END:::keyPrefix---> " + keyPrefix + ":::---> " + + startKey); + } +} \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 8411e3e4c7f3..12442084c98f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1500,9 +1500,19 @@ public List listStatus(OmKeyArgs args, boolean recursive, if (numEntries <= 0) { return fileStatusList; } + + boolean useNewIterator = true; if (isBucketFSOptimized(volName, buckName)) { - return listStatusFSO(args, recursive, startKey, numEntries, - clientAddress); + if (useNewIterator) { + OzoneListStatusHelper statusHelper = + new OzoneListStatusHelper(metadataManager, scmBlockSize, + this::getOzoneFileStatusFSO); + return statusHelper.listStatusFSO(args, recursive, startKey, numEntries, + clientAddress); + } else { + return listStatusFSO(args, recursive, startKey, numEntries, + clientAddress); + } } String volumeName = args.getVolumeName(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java new file mode 100644 index 000000000000..a36b75ab597e --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -0,0 +1,440 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.helpers.*; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Paths; +import java.util.*; + + +public class OzoneListStatusHelper { + + @FunctionalInterface + public interface GetFileStatusHelper { + OzoneFileStatus apply(OmKeyArgs args, String clientAddress, + boolean skipFileNotFoundError) throws IOException; + } + + private static final Logger LOG = + LoggerFactory.getLogger(OzoneListStatusHelper.class); + + private final OMMetadataManager metadataManager; + private final long scmBlockSize; + + GetFileStatusHelper getStatusHelper; + + OzoneListStatusHelper(OMMetadataManager metadataManager, long scmBlockSize, GetFileStatusHelper func) { + this.metadataManager = metadataManager; + this.scmBlockSize = scmBlockSize; + this.getStatusHelper = func; + } + + public List listStatusFSO(OmKeyArgs args, boolean recursive, + String startKey, long numEntries, String clientAddress) + throws IOException { + Preconditions.checkArgument(!recursive); + Preconditions.checkNotNull(args, "Key args can not be null"); + + if (numEntries <= 0) { + return new ArrayList<>(); + } + + final ArrayList fileStatuses = new ArrayList<>(); + + final String volumeName = args.getVolumeName(); + final String bucketName = args.getBucketName(); + final String keyName = args.getKeyName(); + String prefixKey = keyName; + + /** + * a) If the keyname is a file just return one entry + * b) if the keyname is root, then return the value of the bucket + * c) if the keyname is a different bucket than root, fetch the direcoty parent id + * + * if the startkey exists + * a) check the start key is a child ot key, else return emptry list + * b) chekc if the start key is a child chil of keynae, then reset the key to parent of start key + * c) if start key is non existent then seek to the neatest key + * d) if the keyname is not a dir or a file, it can either be invalid name or a prefix path + * in case, this is called as part of listStatus fail as the real dir/file should exist + * else, try to find the parent of keyname and use that as the prefix, use the rest of the path to construct prefix path + */ + + + if (StringUtils.isNotBlank(keyName) && StringUtils.isNotBlank(startKey) && + !OzoneFSUtils.isImmediateChild(keyName, startKey)) { + if (LOG.isDebugEnabled()) { + LOG.debug("StartKey {} is not an immediate child of keyName {}. " + + "Returns empty list", startKey, keyName); + } + return Collections.emptyList(); + } + + String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); + OmBucketInfo omBucketInfo = + metadataManager.getBucketTable().get(bucketKey); + if (omBucketInfo == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("StartKey {} is not an immediate child of keyName {}. " + + "Returns empty list", startKey, keyName); + } + return Collections.emptyList(); + } + + BucketLayout bucketLayout = omBucketInfo.getBucketLayout(); + + OzoneFileStatus fileStatus = getStatusHelper.apply(args, clientAddress, true); + + String dbPrefixKey; + if (fileStatus == null) { + dbPrefixKey = getDbKey(keyName, args, omBucketInfo); + prefixKey = OzoneFSUtils.getParentDir(keyName); + } else { + if (fileStatus.isFile()) { + return Collections.singletonList(fileStatus); + } + long id = getId(fileStatus, omBucketInfo); + dbPrefixKey = metadataManager.getOzonePathKey(id, ""); + } + + String startKeyPrefix = Strings.isNullOrEmpty(startKey) ? "" : getDbKey(startKey, args, omBucketInfo); + + try (MinHeapIterator heapIterator = new MinHeapIterator(metadataManager, dbPrefixKey, bucketLayout, startKeyPrefix)) { + + while (fileStatuses.size() < numEntries && heapIterator.hasNext()) { + HeapEntry entry = heapIterator.next(); + OzoneFileStatus status = entry.getStatus(prefixKey, scmBlockSize, volumeName, bucketName); + LOG.info("returning status:{} keyname:{} startkey:{} numEntries:{}", status, prefixKey, startKey, numEntries); + fileStatuses.add(status); + } + } + + return fileStatuses; + } + + + private String getDbKey(String key, OmKeyArgs args, OmBucketInfo omBucketInfo) throws IOException { + long startKeyParentId; + java.nio.file.Path file = Paths.get(key); + String parent = OzoneFSUtils.getParentDir(key); + + OmKeyArgs startKeyArgs = args.toBuilder() + .setKeyName(parent) + .setSortDatanodesInPipeline(false) + .build(); + OzoneFileStatus fileStatusInfo = getStatusHelper.apply(startKeyArgs, + null, true); + + startKeyParentId = getId(fileStatusInfo, omBucketInfo); + return metadataManager.getOzonePathKey(startKeyParentId, file.getFileName().toString()); + } + + private long getId(OzoneFileStatus fileStatus, OmBucketInfo omBucketInfo) { + if (fileStatus.getKeyInfo() != null) { + return fileStatus.getKeyInfo().getObjectID(); + } else { + // assert root is null + // list root directory. + return omBucketInfo.getObjectID(); + } + } + public enum EntryType { + DIR_CACHE, + FILE_CACHE, + RAW_FILE_DB, + RAW_DIR_DB; + + public boolean isDir() { + switch (this) { + case DIR_CACHE: + case RAW_DIR_DB: + return true; + case FILE_CACHE: + case RAW_FILE_DB: + return false; + default: + throw new IllegalArgumentException(); + } + + } + } + public static class HeapEntry implements Comparable { + private final EntryType entryType; + private final String key; + T value; + + HeapEntry(EntryType entryType, String key, T value) { + this.entryType = entryType; + this.key = key; + this.value = value; + } + + public int compareTo(HeapEntry other) { + return this.key.compareTo(other.key); + } + + public String getKey() { + return key; + } + + public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, String volumeName, String bucketName) { + OmKeyInfo keyInfo; + if (entryType.isDir()) { + OmDirectoryInfo dirInfo = (OmDirectoryInfo) value; + String dirName = OMFileRequest.getAbsolutePath(prefixPath, + dirInfo.getName()); + keyInfo = OMFileRequest.getOmKeyInfo(volumeName, + bucketName, dirInfo, dirName); + } else { + keyInfo = (OmKeyInfo) value; + keyInfo.setFileName(keyInfo.getKeyName()); + String fullKeyPath = OMFileRequest.getAbsolutePath(prefixPath, + keyInfo.getKeyName()); + keyInfo.setKeyName(fullKeyPath); + } + return new OzoneFileStatus(keyInfo, scmBlockSize, entryType.isDir()); + } + } + + public interface OzoneKeyIterator extends Iterator>, Closeable { + + } + + private static class RawIter implements OzoneKeyIterator { + + private final EntryType iterType; + private final String prefixKey; + + private final TableIterator> tableIterator; + private final Set cacheDeletedKeySet; + private HeapEntry currentKey; + + RawIter(EntryType iterType, TableIterator> tableIterator, + String prefixKey, String startKey, Set cacheDeletedKeySet) throws IOException { + this.iterType = iterType; + this.tableIterator = tableIterator; + this.prefixKey = prefixKey; + this.cacheDeletedKeySet = cacheDeletedKeySet; + this.currentKey = null; + if (!StringUtils.isBlank(prefixKey)) { + tableIterator.seek(prefixKey); + } + + if (!StringUtils.isBlank(startKey)) { + tableIterator.seek(startKey); + } + + getNextKey(); + } + + private void getNextKey() throws IOException { + while (tableIterator.hasNext() && currentKey == null) { + Table.KeyValue entry = tableIterator.next(); + String entryKey = entry.getKey(); + if (entryKey.startsWith(prefixKey)) { + if (!cacheDeletedKeySet.contains(entryKey)) { + currentKey = new HeapEntry(iterType, entryKey, entry.getValue()); + } + } else { + break; + } + } + } + + public boolean hasNext() { + try { + getNextKey(); + } catch (Throwable t) { + throw new NoSuchElementException(); + } + return currentKey != null; + } + + public HeapEntry next() { + try { + getNextKey(); + } catch (Throwable t) { + throw new NoSuchElementException(); + } + HeapEntry ret = currentKey; + currentKey = null; + + return ret; + } + + public void close() { + + } + } + + private static class CacheIter< T extends WithParentObjectId> implements OzoneKeyIterator { + private final Set cacheDeletedKeySet; + private final Map cacheKeyMap; + + private final Iterator> cacheCreatedKeyIter; + + private final Iterator, CacheValue>> cacheIter; + + private final String prefixKey; + private final String startKey; + + + private final EntryType entryType; + + public CacheIter(EntryType entryType, Iterator, CacheValue>> cacheIter, + String startKey, String prefixKey) { + this.cacheDeletedKeySet = new TreeSet<>(); + this.cacheKeyMap = new TreeMap<>(); + + this.cacheIter = cacheIter; + this.startKey = startKey; + this.prefixKey = prefixKey; + this.entryType = entryType; + + getCacheValues(); + + cacheCreatedKeyIter = cacheKeyMap.entrySet().iterator(); + } + + private void getCacheValues() { + while (cacheIter.hasNext()) { + Map.Entry, CacheValue> entry = + cacheIter.next(); + String cacheKey = entry.getKey().getCacheKey(); + T cacheOmInfo = entry.getValue().getCacheValue(); + // cacheOmKeyInfo is null if an entry is deleted in cache + if (cacheOmInfo == null) { + cacheDeletedKeySet.add(cacheKey); + continue; + } + + if (StringUtils.isBlank(startKey)) { + // startKey is null or empty, then the seekKeyInDB="1024/" + if (cacheKey.startsWith(prefixKey)) { + cacheKeyMap.put(cacheKey, cacheOmInfo); + } + } else { + // startKey not empty, then the seekKeyInDB="1024/b" and + // seekKeyInDBWithOnlyParentID = "1024/". This is to avoid case of + // parentID with "102444" cache entries. + // Here, it has to list all the keys after "1024/b" and requires >=0 + // string comparison. + if (cacheKey.startsWith(prefixKey) && + cacheKey.compareTo(startKey) >= 0) { + cacheKeyMap.put(cacheKey, cacheOmInfo); + } + } + } + } + + public boolean hasNext() { + return cacheCreatedKeyIter.hasNext(); + } + + public HeapEntry next() { + Map.Entry entry = cacheCreatedKeyIter.next(); + return new HeapEntry(entryType, entry.getKey(), entry.getValue()); + } + + public void close() { + + } + + public Set getDeletedKeySet() { + return cacheDeletedKeySet; + } + } + + public static class MinHeapIterator implements Iterator>, Closeable { + + private final PriorityQueue> minHeap = new PriorityQueue<>(); + private final ArrayList>> iterators = new ArrayList<>(); + + RawIter rawDirIter; + RawIter rawFileIter; + + CacheIter cacheFileIter; + CacheIter cacheDirIter; + + String prefixKey; + + + MinHeapIterator(OMMetadataManager omMetadataManager, String prefixKey, + BucketLayout bucketLayout, String startKey) throws IOException { + + this.prefixKey = prefixKey; + + cacheDirIter = + new CacheIter<>(EntryType.DIR_CACHE, omMetadataManager.getDirectoryTable().cacheIterator(), prefixKey, startKey); + cacheFileIter = + new CacheIter<>(EntryType.FILE_CACHE, omMetadataManager.getKeyTable(bucketLayout).cacheIterator(), prefixKey, startKey); + + rawDirIter = + new RawIter<>(EntryType.RAW_DIR_DB, omMetadataManager.getDirectoryTable().iterator(), + prefixKey, startKey, cacheDirIter.getDeletedKeySet()); + rawFileIter = + new RawIter<>(EntryType.RAW_FILE_DB, omMetadataManager.getKeyTable(bucketLayout).iterator(), + prefixKey, startKey, cacheFileIter.getDeletedKeySet()); + + + iterators.add(EntryType.DIR_CACHE.ordinal(), cacheDirIter); + iterators.add(EntryType.FILE_CACHE.ordinal(), cacheFileIter); + iterators.add(EntryType.RAW_FILE_DB.ordinal(), rawFileIter); + iterators.add(EntryType.RAW_DIR_DB.ordinal(), rawDirIter); + insertFirstElement(); + + } + + public void insertFirstElement() { + for (Iterator> iter : iterators) { + if (iter.hasNext()) { + minHeap.add(iter.next()); + } + } + } + + public boolean hasNext() { + return !minHeap.isEmpty(); + } + + public HeapEntry next() { + HeapEntry heapEntry = minHeap.remove(); + Iterator> iter = iterators.get(heapEntry.entryType.ordinal()); + if (iter.hasNext()) { + minHeap.add(iter.next()); + } + + return heapEntry; + } + + public void close() throws IOException { + } + } +} From 401eae04ed38afe7627b7891f47aca8a743c0e1b Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Mon, 23 May 2022 21:53:28 +0530 Subject: [PATCH 02/13] fix checktyle --- .../hadoop/ozone/om/TestListStatus.java | 36 +++--- .../ozone/om/OzoneListStatusHelper.java | 103 +++++++++++++----- 2 files changed, 98 insertions(+), 41 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java index 982cd117281a..a45724f9f1f4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java @@ -23,15 +23,23 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.TestDataUtil; -import org.apache.hadoop.ozone.client.*; +import org.apache.hadoop.ozone.client.BucketArgs; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; -import org.junit.*; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.AfterClass; import org.junit.rules.Timeout; import java.io.IOException; -import java.util.*; +import java.util.UUID; +import java.util.List; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; @@ -151,11 +159,12 @@ public void testSortedListStatus() throws Exception { // checkKeyList("b", "b5", 100, 2); } - private static void createFile(OzoneBucket bucket, String keyName) throws IOException { + private static void createFile(OzoneBucket bucket, String keyName) + throws IOException { try (OzoneOutputStream oos = bucket.createFile(keyName, 0, RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE), true, false)) { - + oos.flush(); } } private static void buildNameSpaceTree(OzoneBucket ozoneBucket) @@ -168,20 +177,21 @@ private static void buildNameSpaceTree(OzoneBucket ozoneBucket) createFile(ozoneBucket, "/a10"); ozoneBucket.createDirectory("/a1/a11"); - createFile(ozoneBucket,"/a1/a12"); - createFile(ozoneBucket,"/a1/a13"); + createFile(ozoneBucket, "/a1/a12"); + createFile(ozoneBucket, "/a1/a13"); ozoneBucket.createDirectory("/a3/a31"); - createFile(ozoneBucket,"/a3/a32"); + createFile(ozoneBucket, "/a3/a32"); - createFile(ozoneBucket,"/b1"); - createFile(ozoneBucket,"/b2"); - createFile(ozoneBucket,"/b7"); - createFile(ozoneBucket,"/b8"); + createFile(ozoneBucket, "/b1"); + createFile(ozoneBucket, "/b2"); + createFile(ozoneBucket, "/b7"); + createFile(ozoneBucket, "/b8"); } private void checkKeyList(String keyPrefix, String startKey, - long numEntries, int expectedNumKeys) throws Exception { + long numEntries, int expectedNumKeys) + throws Exception { List statuses = fsoOzoneBucket.listStatus(keyPrefix, false, startKey, numEntries); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index a36b75ab597e..796349b51ec6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -23,7 +23,14 @@ import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; -import org.apache.hadoop.ozone.om.helpers.*; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.WithParentObjectId; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +38,19 @@ import java.io.Closeable; import java.io.IOException; import java.nio.file.Paths; -import java.util.*; +import java.util.List; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.TreeMap; +import java.util.Iterator; +import java.util.PriorityQueue; +import java.util.NoSuchElementException; + + +import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; public class OzoneListStatusHelper { @@ -126,7 +145,10 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, String startKeyPrefix = Strings.isNullOrEmpty(startKey) ? "" : getDbKey(startKey, args, omBucketInfo); - try (MinHeapIterator heapIterator = new MinHeapIterator(metadataManager, dbPrefixKey, bucketLayout, startKeyPrefix)) { + + try (MinHeapIterator heapIterator = + new MinHeapIterator(metadataManager, dbPrefixKey, bucketLayout, + startKeyPrefix, volumeName, bucketName)) { while (fileStatuses.size() < numEntries && heapIterator.hasNext()) { HeapEntry entry = heapIterator.next(); @@ -151,7 +173,7 @@ private String getDbKey(String key, OmKeyArgs args, OmBucketInfo omBucketInfo) t .build(); OzoneFileStatus fileStatusInfo = getStatusHelper.apply(startKeyArgs, null, true); - + Preconditions.checkNotNull(fileStatusInfo); startKeyParentId = getId(fileStatusInfo, omBucketInfo); return metadataManager.getOzonePathKey(startKeyParentId, file.getFileName().toString()); } @@ -200,6 +222,11 @@ public int compareTo(HeapEntry other) { return this.key.compareTo(other.key); } + public boolean equals(Object other) { + HeapEntry that = (HeapEntry) other; + return this.key.equals(that.key); + } + public String getKey() { return key; } @@ -227,17 +254,21 @@ public interface OzoneKeyIterator extends Iterator implements OzoneKeyIterator { + private static class RawIter + implements OzoneKeyIterator { private final EntryType iterType; private final String prefixKey; - private final TableIterator> tableIterator; + private final TableIterator> tableIterator; private final Set cacheDeletedKeySet; private HeapEntry currentKey; - RawIter(EntryType iterType, TableIterator> tableIterator, - String prefixKey, String startKey, Set cacheDeletedKeySet) throws IOException { + RawIter(EntryType iterType, TableIterator> tableIterator, + String prefixKey, String startKey, + Set cacheDeletedKeySet) throws IOException { this.iterType = iterType; this.tableIterator = tableIterator; this.prefixKey = prefixKey; @@ -294,13 +325,16 @@ public void close() { } } - private static class CacheIter< T extends WithParentObjectId> implements OzoneKeyIterator { + private static class CacheIter< T extends WithParentObjectId> + implements OzoneKeyIterator { private final Set cacheDeletedKeySet; private final Map cacheKeyMap; - private final Iterator> cacheCreatedKeyIter; + private final Iterator> + cacheCreatedKeyIter; - private final Iterator, CacheValue>> cacheIter; + private final Iterator, CacheValue>> + cacheIter; private final String prefixKey; private final String startKey; @@ -308,8 +342,8 @@ private static class CacheIter< T extends WithParentObjectId> implements OzoneKe private final EntryType entryType; - public CacheIter(EntryType entryType, Iterator, CacheValue>> cacheIter, - String startKey, String prefixKey) { + public CacheIter(EntryType entryType, Iterator, + CacheValue>> cacheIter, String startKey, String prefixKey) { this.cacheDeletedKeySet = new TreeSet<>(); this.cacheKeyMap = new TreeMap<>(); @@ -372,10 +406,13 @@ public Set getDeletedKeySet() { } } - public static class MinHeapIterator implements Iterator>, Closeable { + public static class MinHeapIterator implements + Iterator>, Closeable { - private final PriorityQueue> minHeap = new PriorityQueue<>(); - private final ArrayList>> iterators = new ArrayList<>(); + private final PriorityQueue> minHeap = new PriorityQueue<>(); + private final ArrayList>> iterators = new ArrayList<>(); RawIter rawDirIter; RawIter rawFileIter; @@ -383,26 +420,34 @@ public static class MinHeapIterator implements Iterator cacheFileIter; CacheIter cacheDirIter; - String prefixKey; - - MinHeapIterator(OMMetadataManager omMetadataManager, String prefixKey, - BucketLayout bucketLayout, String startKey) throws IOException { - - this.prefixKey = prefixKey; + BucketLayout bucketLayout, String startKey, + String volumeName, String bucketName) throws IOException { + omMetadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, + bucketName); cacheDirIter = - new CacheIter<>(EntryType.DIR_CACHE, omMetadataManager.getDirectoryTable().cacheIterator(), prefixKey, startKey); + new CacheIter<>(EntryType.DIR_CACHE, + omMetadataManager.getDirectoryTable().cacheIterator(), + startKey, prefixKey); + cacheFileIter = - new CacheIter<>(EntryType.FILE_CACHE, omMetadataManager.getKeyTable(bucketLayout).cacheIterator(), prefixKey, startKey); + new CacheIter<>(EntryType.FILE_CACHE, + omMetadataManager.getKeyTable(bucketLayout).cacheIterator(), + startKey, prefixKey); rawDirIter = - new RawIter<>(EntryType.RAW_DIR_DB, omMetadataManager.getDirectoryTable().iterator(), + new RawIter<>(EntryType.RAW_DIR_DB, + omMetadataManager.getDirectoryTable().iterator(), prefixKey, startKey, cacheDirIter.getDeletedKeySet()); + rawFileIter = - new RawIter<>(EntryType.RAW_FILE_DB, omMetadataManager.getKeyTable(bucketLayout).iterator(), + new RawIter<>(EntryType.RAW_FILE_DB, + omMetadataManager.getKeyTable(bucketLayout).iterator(), prefixKey, startKey, cacheFileIter.getDeletedKeySet()); + omMetadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, + bucketName); iterators.add(EntryType.DIR_CACHE.ordinal(), cacheDirIter); iterators.add(EntryType.FILE_CACHE.ordinal(), cacheFileIter); @@ -413,7 +458,8 @@ public static class MinHeapIterator implements Iterator> iter : iterators) { + for (Iterator> iter : + iterators) { if (iter.hasNext()) { minHeap.add(iter.next()); } @@ -426,7 +472,8 @@ public boolean hasNext() { public HeapEntry next() { HeapEntry heapEntry = minHeap.remove(); - Iterator> iter = iterators.get(heapEntry.entryType.ordinal()); + Iterator> iter = + iterators.get(heapEntry.entryType.ordinal()); if (iter.hasNext()) { minHeap.add(iter.next()); } From 9ad667092a6a58aef62757a0ec7cd9606bcd8389 Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Tue, 24 May 2022 09:46:52 +0530 Subject: [PATCH 03/13] fix test issues --- .../ozone/om/OzoneListStatusHelper.java | 126 +++++++++++------- 1 file changed, 80 insertions(+), 46 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index 796349b51ec6..ed1e2719ffdd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -38,23 +38,19 @@ import java.io.Closeable; import java.io.IOException; import java.nio.file.Paths; -import java.util.List; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.TreeMap; -import java.util.Iterator; -import java.util.PriorityQueue; -import java.util.NoSuchElementException; +import java.util.*; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; - +/** + * Helper class for fetching List Status for a path. + */ public class OzoneListStatusHelper { + /** + * Interface to get the File Status. + */ @FunctionalInterface public interface GetFileStatusHelper { OzoneFileStatus apply(OmKeyArgs args, String clientAddress, @@ -67,16 +63,18 @@ OzoneFileStatus apply(OmKeyArgs args, String clientAddress, private final OMMetadataManager metadataManager; private final long scmBlockSize; - GetFileStatusHelper getStatusHelper; + private final GetFileStatusHelper getStatusHelper; - OzoneListStatusHelper(OMMetadataManager metadataManager, long scmBlockSize, GetFileStatusHelper func) { + OzoneListStatusHelper(OMMetadataManager metadataManager, long scmBlockSize, + GetFileStatusHelper func) { this.metadataManager = metadataManager; this.scmBlockSize = scmBlockSize; this.getStatusHelper = func; } - public List listStatusFSO(OmKeyArgs args, boolean recursive, - String startKey, long numEntries, String clientAddress) + public List listStatusFSO(OmKeyArgs args, + boolean recursive, String startKey, long numEntries, + String clientAddress) throws IOException { Preconditions.checkArgument(!recursive); Preconditions.checkNotNull(args, "Key args can not be null"); @@ -85,8 +83,6 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, return new ArrayList<>(); } - final ArrayList fileStatuses = new ArrayList<>(); - final String volumeName = args.getVolumeName(); final String bucketName = args.getBucketName(); final String keyName = args.getKeyName(); @@ -129,7 +125,8 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, BucketLayout bucketLayout = omBucketInfo.getBucketLayout(); - OzoneFileStatus fileStatus = getStatusHelper.apply(args, clientAddress, true); + OzoneFileStatus fileStatus = + getStatusHelper.apply(args, clientAddress, true); String dbPrefixKey; if (fileStatus == null) { @@ -143,26 +140,32 @@ public List listStatusFSO(OmKeyArgs args, boolean recursive, dbPrefixKey = metadataManager.getOzonePathKey(id, ""); } - String startKeyPrefix = Strings.isNullOrEmpty(startKey) ? "" : getDbKey(startKey, args, omBucketInfo); + String startKeyPrefix = + Strings.isNullOrEmpty(startKey) ? "" : + getDbKey(startKey, args, omBucketInfo); + TreeMap cacheKeyMap = new TreeMap<>(); try (MinHeapIterator heapIterator = new MinHeapIterator(metadataManager, dbPrefixKey, bucketLayout, startKeyPrefix, volumeName, bucketName)) { - while (fileStatuses.size() < numEntries && heapIterator.hasNext()) { + while (cacheKeyMap.size() < numEntries && heapIterator.hasNext()) { HeapEntry entry = heapIterator.next(); - OzoneFileStatus status = entry.getStatus(prefixKey, scmBlockSize, volumeName, bucketName); - LOG.info("returning status:{} keyname:{} startkey:{} numEntries:{}", status, prefixKey, startKey, numEntries); - fileStatuses.add(status); + OzoneFileStatus status = entry.getStatus(prefixKey, + scmBlockSize, volumeName, bucketName); + LOG.info("returning status:{} keyname:{} startkey:{} numEntries:{}", + status, prefixKey, startKey, numEntries); + cacheKeyMap.put(entry.getKey(), status); } } - return fileStatuses; + return new ArrayList<>(cacheKeyMap.values()); } - private String getDbKey(String key, OmKeyArgs args, OmBucketInfo omBucketInfo) throws IOException { + private String getDbKey(String key, OmKeyArgs args, + OmBucketInfo omBucketInfo) throws IOException { long startKeyParentId; java.nio.file.Path file = Paths.get(key); String parent = OzoneFSUtils.getParentDir(key); @@ -175,7 +178,8 @@ private String getDbKey(String key, OmKeyArgs args, OmBucketInfo omBucketInfo) t null, true); Preconditions.checkNotNull(fileStatusInfo); startKeyParentId = getId(fileStatusInfo, omBucketInfo); - return metadataManager.getOzonePathKey(startKeyParentId, file.getFileName().toString()); + return metadataManager. + getOzonePathKey(startKeyParentId, file.getFileName().toString()); } private long getId(OzoneFileStatus fileStatus, OmBucketInfo omBucketInfo) { @@ -187,6 +191,10 @@ private long getId(OzoneFileStatus fileStatus, OmBucketInfo omBucketInfo) { return omBucketInfo.getObjectID(); } } + + /** + * Enum of types of entries + */ public enum EntryType { DIR_CACHE, FILE_CACHE, @@ -195,22 +203,26 @@ public enum EntryType { public boolean isDir() { switch (this) { - case DIR_CACHE: - case RAW_DIR_DB: - return true; - case FILE_CACHE: - case RAW_FILE_DB: - return false; - default: - throw new IllegalArgumentException(); - } - + case DIR_CACHE: + case RAW_DIR_DB: + return true; + case FILE_CACHE: + case RAW_FILE_DB: + return false; + default: + throw new IllegalArgumentException(); + } } } - public static class HeapEntry implements Comparable { + + /** + * Entry which is added to heap + * @param + */ + private static class HeapEntry implements Comparable { private final EntryType entryType; private final String key; - T value; + private final T value; HeapEntry(EntryType entryType, String key, T value) { this.entryType = entryType; @@ -223,15 +235,29 @@ public int compareTo(HeapEntry other) { } public boolean equals(Object other) { + if (other == null) { + return false; + } + + if (! (other instanceof HeapEntry)) { + return false; + } + + HeapEntry that = (HeapEntry) other; return this.key.equals(that.key); } + public int hashCode() { + return key.hashCode(); + } + public String getKey() { return key; } - public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, String volumeName, String bucketName) { + public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, + String volumeName, String bucketName) { OmKeyInfo keyInfo; if (entryType.isDir()) { OmDirectoryInfo dirInfo = (OmDirectoryInfo) value; @@ -250,8 +276,8 @@ public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, String vo } } - public interface OzoneKeyIterator extends Iterator>, Closeable { - + public interface OzoneKeyIterator extends + Iterator>, Closeable { } private static class RawIter @@ -325,6 +351,10 @@ public void close() { } } + /** + * Iterator for Cache for the database. + * @param + */ private static class CacheIter< T extends WithParentObjectId> implements OzoneKeyIterator { private final Set cacheDeletedKeySet; @@ -406,7 +436,11 @@ public Set getDeletedKeySet() { } } - public static class MinHeapIterator implements + /** + * Implement a min heap iterator to find the smaller + * lexicographically sorted string. + */ + private static class MinHeapIterator implements Iterator>, Closeable { private final PriorityQueue>> iterators = new ArrayList<>(); - RawIter rawDirIter; - RawIter rawFileIter; + private final RawIter rawDirIter; + private final RawIter rawFileIter; - CacheIter cacheFileIter; - CacheIter cacheDirIter; + private final CacheIter cacheFileIter; + private final CacheIter cacheDirIter; MinHeapIterator(OMMetadataManager omMetadataManager, String prefixKey, BucketLayout bucketLayout, String startKey, From 2eeed58d5c6884b7d0e2720bbc543461c421bbba Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Tue, 24 May 2022 14:09:17 +0530 Subject: [PATCH 04/13] fix chckstyle --- .../ozone/om/OzoneListStatusHelper.java | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index ed1e2719ffdd..c1ea14ff012c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -38,7 +38,16 @@ import java.io.Closeable; import java.io.IOException; import java.nio.file.Paths; -import java.util.*; +import java.util.Collections; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.TreeMap; +import java.util.Map; +import java.util.TreeSet; +import java.util.PriorityQueue; +import java.util.Set; +import java.util.NoSuchElementException; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; @@ -91,15 +100,20 @@ public List listStatusFSO(OmKeyArgs args, /** * a) If the keyname is a file just return one entry * b) if the keyname is root, then return the value of the bucket - * c) if the keyname is a different bucket than root, fetch the direcoty parent id + * c) if the keyname is a different bucket than root, + * fetch the direcoty parent id * * if the startkey exists * a) check the start key is a child ot key, else return emptry list - * b) chekc if the start key is a child chil of keynae, then reset the key to parent of start key + * b) chekc if the start key is a child chil of keynae, + * then reset the key to parent of start key * c) if start key is non existent then seek to the neatest key - * d) if the keyname is not a dir or a file, it can either be invalid name or a prefix path - * in case, this is called as part of listStatus fail as the real dir/file should exist - * else, try to find the parent of keyname and use that as the prefix, use the rest of the path to construct prefix path + * d) if the keyname is not a dir or a file, it can either be + * invalid name or a prefix path + * in case, this is called as part of listStatus fail as the + * real dir/file should exist + * else, try to find the parent of keyname and use that as the prefix, + * use the rest of the path to construct prefix path */ @@ -193,7 +207,7 @@ private long getId(OzoneFileStatus fileStatus, OmBucketInfo omBucketInfo) { } /** - * Enum of types of entries + * Enum of types of entries. */ public enum EntryType { DIR_CACHE, @@ -211,15 +225,16 @@ public boolean isDir() { return false; default: throw new IllegalArgumentException(); - } + } } } /** - * Entry which is added to heap + * Entry which is added to heap. * @param */ - private static class HeapEntry implements Comparable { + private static class HeapEntry + implements Comparable { private final EntryType entryType; private final String key; private final T value; @@ -239,7 +254,7 @@ public boolean equals(Object other) { return false; } - if (! (other instanceof HeapEntry)) { + if (!(other instanceof HeapEntry)) { return false; } @@ -276,10 +291,17 @@ public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, } } + /** + * Iterator class for Ozone keys + */ public interface OzoneKeyIterator extends Iterator>, Closeable { } + /** + * Raw iterator over db tables. + * @param + */ private static class RawIter implements OzoneKeyIterator { @@ -372,7 +394,7 @@ private static class CacheIter< T extends WithParentObjectId> private final EntryType entryType; - public CacheIter(EntryType entryType, Iterator, + CacheIter(EntryType entryType, Iterator, CacheValue>> cacheIter, String startKey, String prefixKey) { this.cacheDeletedKeySet = new TreeSet<>(); this.cacheKeyMap = new TreeMap<>(); From 64bf12687b2b6f6b6c48b19880eafa04aaa9213a Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Tue, 24 May 2022 21:07:15 +0530 Subject: [PATCH 05/13] fix test failures --- .../hadoop/ozone/om/KeyManagerImpl.java | 7 ++-- .../ozone/om/OzoneListStatusHelper.java | 33 +++++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 12442084c98f..2a3f3871de8d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1507,8 +1507,11 @@ public List listStatus(OmKeyArgs args, boolean recursive, OzoneListStatusHelper statusHelper = new OzoneListStatusHelper(metadataManager, scmBlockSize, this::getOzoneFileStatusFSO); - return statusHelper.listStatusFSO(args, recursive, startKey, numEntries, - clientAddress); + Map cacheFileMap = new HashMap<>(); + Map cacheDirMap = new HashMap<>(); + statusHelper.listStatusFSO(args, recursive, startKey, numEntries, + clientAddress, cacheDirMap, cacheFileMap); + return buildFinalStatusList(cacheFileMap, cacheDirMap, args, clientAddress); } else { return listStatusFSO(args, recursive, startKey, numEntries, clientAddress); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index c1ea14ff012c..57c5f8fea699 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -81,15 +81,15 @@ OzoneFileStatus apply(OmKeyArgs args, String clientAddress, this.getStatusHelper = func; } - public List listStatusFSO(OmKeyArgs args, + public void listStatusFSO(OmKeyArgs args, boolean recursive, String startKey, long numEntries, - String clientAddress) + String clientAddress, Map cacheDirMap, Map cacheFileMap) throws IOException { Preconditions.checkArgument(!recursive); Preconditions.checkNotNull(args, "Key args can not be null"); if (numEntries <= 0) { - return new ArrayList<>(); + return; } final String volumeName = args.getVolumeName(); @@ -123,7 +123,7 @@ public List listStatusFSO(OmKeyArgs args, LOG.debug("StartKey {} is not an immediate child of keyName {}. " + "Returns empty list", startKey, keyName); } - return Collections.emptyList(); + return; } String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); @@ -134,7 +134,7 @@ public List listStatusFSO(OmKeyArgs args, LOG.debug("StartKey {} is not an immediate child of keyName {}. " + "Returns empty list", startKey, keyName); } - return Collections.emptyList(); + return; } BucketLayout bucketLayout = omBucketInfo.getBucketLayout(); @@ -148,8 +148,10 @@ public List listStatusFSO(OmKeyArgs args, prefixKey = OzoneFSUtils.getParentDir(keyName); } else { if (fileStatus.isFile()) { - return Collections.singletonList(fileStatus); + cacheFileMap.put(fileStatus.getPath(), fileStatus); + return; } + long id = getId(fileStatus, omBucketInfo); dbPrefixKey = metadataManager.getOzonePathKey(id, ""); } @@ -158,23 +160,25 @@ public List listStatusFSO(OmKeyArgs args, Strings.isNullOrEmpty(startKey) ? "" : getDbKey(startKey, args, omBucketInfo); - TreeMap cacheKeyMap = new TreeMap<>(); - try (MinHeapIterator heapIterator = new MinHeapIterator(metadataManager, dbPrefixKey, bucketLayout, startKeyPrefix, volumeName, bucketName)) { - while (cacheKeyMap.size() < numEntries && heapIterator.hasNext()) { + int count = 0; + while (count < numEntries && heapIterator.hasNext()) { HeapEntry entry = heapIterator.next(); OzoneFileStatus status = entry.getStatus(prefixKey, - scmBlockSize, volumeName, bucketName); + scmBlockSize, volumeName, bucketName, args); LOG.info("returning status:{} keyname:{} startkey:{} numEntries:{}", status, prefixKey, startKey, numEntries); - cacheKeyMap.put(entry.getKey(), status); + if (entry.entryType.isDir()) { + cacheDirMap.put(status.getPath(), status); + } else { + cacheFileMap.put(status.getPath(), status); + } + count++; } } - - return new ArrayList<>(cacheKeyMap.values()); } @@ -272,7 +276,8 @@ public String getKey() { } public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, - String volumeName, String bucketName) { + String volumeName, String bucketName, + OmKeyArgs args) { OmKeyInfo keyInfo; if (entryType.isDir()) { OmDirectoryInfo dirInfo = (OmDirectoryInfo) value; From 955ecd2d80561eab4a601ab937f23d830b1ef35e Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Tue, 24 May 2022 22:47:22 +0530 Subject: [PATCH 06/13] fix failure --- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 3 ++- .../hadoop/ozone/om/OzoneListStatusHelper.java | 13 +++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 2a3f3871de8d..86fc5fc34cb5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1511,7 +1511,8 @@ public List listStatus(OmKeyArgs args, boolean recursive, Map cacheDirMap = new HashMap<>(); statusHelper.listStatusFSO(args, recursive, startKey, numEntries, clientAddress, cacheDirMap, cacheFileMap); - return buildFinalStatusList(cacheFileMap, cacheDirMap, args, clientAddress); + return buildFinalStatusList(cacheFileMap, cacheDirMap, + args, clientAddress); } else { return listStatusFSO(args, recursive, startKey, numEntries, clientAddress); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index 57c5f8fea699..78ea311e61a2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -37,11 +37,8 @@ import java.io.Closeable; import java.io.IOException; -import java.nio.file.Paths; -import java.util.Collections; import java.util.ArrayList; import java.util.Iterator; -import java.util.List; import java.util.TreeMap; import java.util.Map; import java.util.TreeSet; @@ -83,7 +80,8 @@ OzoneFileStatus apply(OmKeyArgs args, String clientAddress, public void listStatusFSO(OmKeyArgs args, boolean recursive, String startKey, long numEntries, - String clientAddress, Map cacheDirMap, Map cacheFileMap) + String clientAddress, Map cacheDirMap, + Map cacheFileMap) throws IOException { Preconditions.checkArgument(!recursive); Preconditions.checkNotNull(args, "Key args can not be null"); @@ -176,7 +174,7 @@ public void listStatusFSO(OmKeyArgs args, } else { cacheFileMap.put(status.getPath(), status); } - count++; + count++; } } } @@ -185,7 +183,6 @@ public void listStatusFSO(OmKeyArgs args, private String getDbKey(String key, OmKeyArgs args, OmBucketInfo omBucketInfo) throws IOException { long startKeyParentId; - java.nio.file.Path file = Paths.get(key); String parent = OzoneFSUtils.getParentDir(key); OmKeyArgs startKeyArgs = args.toBuilder() @@ -197,7 +194,7 @@ private String getDbKey(String key, OmKeyArgs args, Preconditions.checkNotNull(fileStatusInfo); startKeyParentId = getId(fileStatusInfo, omBucketInfo); return metadataManager. - getOzonePathKey(startKeyParentId, file.getFileName().toString()); + getOzonePathKey(startKeyParentId, OzoneFSUtils.getFileName(key)); } private long getId(OzoneFileStatus fileStatus, OmBucketInfo omBucketInfo) { @@ -297,7 +294,7 @@ public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, } /** - * Iterator class for Ozone keys + * Iterator class for Ozone keys. */ public interface OzoneKeyIterator extends Iterator>, Closeable { From f69e75a9581b4c620e616f71b644a13da112ad45 Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Wed, 25 May 2022 01:02:45 +0530 Subject: [PATCH 07/13] fix more tests --- .../hadoop/ozone/om/KeyManagerImpl.java | 38 ++++++++-- .../ozone/om/OzoneListStatusHelper.java | 71 +++++++++++-------- 2 files changed, 76 insertions(+), 33 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 86fc5fc34cb5..60537583d17e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1507,11 +1507,10 @@ public List listStatus(OmKeyArgs args, boolean recursive, OzoneListStatusHelper statusHelper = new OzoneListStatusHelper(metadataManager, scmBlockSize, this::getOzoneFileStatusFSO); - Map cacheFileMap = new HashMap<>(); - Map cacheDirMap = new HashMap<>(); - statusHelper.listStatusFSO(args, recursive, startKey, numEntries, - clientAddress, cacheDirMap, cacheFileMap); - return buildFinalStatusList(cacheFileMap, cacheDirMap, + Collection statuses = + statusHelper.listStatusFSO(args, recursive, startKey, numEntries, + clientAddress); + return buildFinalStatusList(statuses, args, clientAddress); } else { return listStatusFSO(args, recursive, startKey, numEntries, @@ -1903,6 +1902,34 @@ private List buildFinalStatusList( fileStatusFinalList.add(fileStatus); } + return sortPipelineInfo(fileStatusFinalList, keyInfoList, + omKeyArgs, clientAddress); + } + + private List buildFinalStatusList( + Collection statusesCollection, OmKeyArgs omKeyArgs, + String clientAddress) + throws IOException { + List fileStatusFinalList = new ArrayList<>(); + List keyInfoList = new ArrayList<>(); + + for (OzoneFileStatus fileStatus : statusesCollection) { + if (fileStatus.isFile()) { + keyInfoList.add(fileStatus.getKeyInfo()); + } + fileStatusFinalList.add(fileStatus); + } + + return sortPipelineInfo(fileStatusFinalList, keyInfoList, + omKeyArgs, clientAddress); + } + + + private List sortPipelineInfo( + List fileStatusFinalList, List keyInfoList, + OmKeyArgs omKeyArgs, String clientAddress) throws IOException { + + if (omKeyArgs.getLatestVersionLocation()) { slimLocationVersion(keyInfoList.toArray(new OmKeyInfo[0])); } @@ -1918,6 +1945,7 @@ private List buildFinalStatusList( return fileStatusFinalList; } + /*** * Build files, directories and marked for deleted entries from dir/file * cache. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index 78ea311e61a2..6c0fe62250d2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -45,6 +45,8 @@ import java.util.PriorityQueue; import java.util.Set; import java.util.NoSuchElementException; +import java.util.Collection; +import java.util.Collections; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; @@ -78,21 +80,20 @@ OzoneFileStatus apply(OmKeyArgs args, String clientAddress, this.getStatusHelper = func; } - public void listStatusFSO(OmKeyArgs args, + public Collection listStatusFSO(OmKeyArgs args, boolean recursive, String startKey, long numEntries, - String clientAddress, Map cacheDirMap, - Map cacheFileMap) + String clientAddress) throws IOException { Preconditions.checkArgument(!recursive); Preconditions.checkNotNull(args, "Key args can not be null"); if (numEntries <= 0) { - return; + return new ArrayList<>(); } final String volumeName = args.getVolumeName(); final String bucketName = args.getBucketName(); - final String keyName = args.getKeyName(); + String keyName = args.getKeyName(); String prefixKey = keyName; /** @@ -115,15 +116,6 @@ public void listStatusFSO(OmKeyArgs args, */ - if (StringUtils.isNotBlank(keyName) && StringUtils.isNotBlank(startKey) && - !OzoneFSUtils.isImmediateChild(keyName, startKey)) { - if (LOG.isDebugEnabled()) { - LOG.debug("StartKey {} is not an immediate child of keyName {}. " + - "Returns empty list", startKey, keyName); - } - return; - } - String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); OmBucketInfo omBucketInfo = metadataManager.getBucketTable().get(bucketKey); @@ -132,13 +124,38 @@ public void listStatusFSO(OmKeyArgs args, LOG.debug("StartKey {} is not an immediate child of keyName {}. " + "Returns empty list", startKey, keyName); } - return; + return new ArrayList<>(); } - BucketLayout bucketLayout = omBucketInfo.getBucketLayout(); + boolean pathNameChanged = false; + if (StringUtils.isNotBlank(startKey)) { + if (StringUtils.isNotBlank(keyName)) { + if ( !OzoneFSUtils.isImmediateChild(keyName, startKey)) { + if (LOG.isDebugEnabled()) { + LOG.debug("StartKey {} is not an immediate child of keyName {}. " + + "Returns empty list", startKey, keyName); + } + return new ArrayList<>(); + } + } else { + keyName = OzoneFSUtils.getParentDir(startKey); + prefixKey = keyName; + pathNameChanged = true; + } + } - OzoneFileStatus fileStatus = - getStatusHelper.apply(args, clientAddress, true); + OzoneFileStatus fileStatus = null; + if (pathNameChanged) { + OmKeyArgs startKeyArgs = args.toBuilder() + .setKeyName(keyName) + .setSortDatanodesInPipeline(false) + .build(); + fileStatus = getStatusHelper.apply(startKeyArgs, + null, true); + } else { + fileStatus = + getStatusHelper.apply(args, clientAddress, true); + } String dbPrefixKey; if (fileStatus == null) { @@ -146,8 +163,7 @@ public void listStatusFSO(OmKeyArgs args, prefixKey = OzoneFSUtils.getParentDir(keyName); } else { if (fileStatus.isFile()) { - cacheFileMap.put(fileStatus.getPath(), fileStatus); - return; + return Collections.singletonList(fileStatus); } long id = getId(fileStatus, omBucketInfo); @@ -158,25 +174,24 @@ public void listStatusFSO(OmKeyArgs args, Strings.isNullOrEmpty(startKey) ? "" : getDbKey(startKey, args, omBucketInfo); + TreeMap map = new TreeMap<>(); + + BucketLayout bucketLayout = omBucketInfo.getBucketLayout(); try (MinHeapIterator heapIterator = new MinHeapIterator(metadataManager, dbPrefixKey, bucketLayout, startKeyPrefix, volumeName, bucketName)) { - int count = 0; - while (count < numEntries && heapIterator.hasNext()) { + while (map.size() < numEntries && heapIterator.hasNext()) { HeapEntry entry = heapIterator.next(); OzoneFileStatus status = entry.getStatus(prefixKey, scmBlockSize, volumeName, bucketName, args); LOG.info("returning status:{} keyname:{} startkey:{} numEntries:{}", status, prefixKey, startKey, numEntries); - if (entry.entryType.isDir()) { - cacheDirMap.put(status.getPath(), status); - } else { - cacheFileMap.put(status.getPath(), status); - } - count++; + map.put(entry.key, status); } } + + return map.values(); } From 33868a1b5c194acaedb23e1d2be322ef62abd1e8 Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Wed, 25 May 2022 01:11:20 +0530 Subject: [PATCH 08/13] add sorted check --- .../org/apache/hadoop/ozone/om/TestListStatus.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java index a45724f9f1f4..fccfdf3da8ce 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java @@ -38,6 +38,7 @@ import org.junit.rules.Timeout; import java.io.IOException; +import java.util.Collections; import java.util.UUID; import java.util.List; @@ -199,9 +200,18 @@ private void checkKeyList(String keyPrefix, String startKey, System.out.println("BEGIN:::keyPrefix---> " + keyPrefix + ":::---> " + startKey); - for (OzoneFileStatus st : statuses) { - System.out.println("status:" + st); + + for (int i = 0; i < statuses.size() - 1; i++) { + OzoneFileStatus stCurr = statuses.get(i); + OzoneFileStatus stNext = statuses.get(i + 1); + + System.out.println("status:" + stCurr); + Assert.assertTrue(stCurr.getPath().compareTo(stNext.getPath()) < 0); } + + OzoneFileStatus stNext = statuses.get(statuses.size() - 1); + System.out.println("status:" + stNext); + System.out.println("END:::keyPrefix---> " + keyPrefix + ":::---> " + startKey); } From b3002d4b7c1acf012965c26248244b57820dabef Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Wed, 25 May 2022 08:03:31 +0530 Subject: [PATCH 09/13] fix list Status --- .../apache/hadoop/ozone/om/TestListStatus.java | 18 ++++++++++-------- .../hadoop/ozone/om/OzoneListStatusHelper.java | 8 ++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java index fccfdf3da8ce..a372b445ef89 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java @@ -38,7 +38,6 @@ import org.junit.rules.Timeout; import java.io.IOException; -import java.util.Collections; import java.util.UUID; import java.util.List; @@ -153,11 +152,12 @@ public void testSortedListStatus() throws Exception { // f) check with non existing start key>>> checkKeyList("", "a7", 100, 6); - // g) check if half prefix works <<<< - // checkKeyList("b", "", 100, 4); - - // h) check half prefix with non-existing start key - // checkKeyList("b", "b5", 100, 2); + // TODO: Enable the following test after listKeys changes +// // g) check if half prefix works <<<< +// checkKeyList("b", "", 100, 4); +// +// // h) check half prefix with non-existing start key +// checkKeyList("b", "b5", 100, 2); } private static void createFile(OzoneBucket bucket, String keyName) @@ -209,8 +209,10 @@ private void checkKeyList(String keyPrefix, String startKey, Assert.assertTrue(stCurr.getPath().compareTo(stNext.getPath()) < 0); } - OzoneFileStatus stNext = statuses.get(statuses.size() - 1); - System.out.println("status:" + stNext); + if (!statuses.isEmpty()) { + OzoneFileStatus stNext = statuses.get(statuses.size() - 1); + System.out.println("status:" + stNext); + } System.out.println("END:::keyPrefix---> " + keyPrefix + ":::---> " + startKey); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index 6c0fe62250d2..5c019a80e810 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -91,6 +91,7 @@ public Collection listStatusFSO(OmKeyArgs args, return new ArrayList<>(); } + boolean listKeysMode = false; final String volumeName = args.getVolumeName(); final String bucketName = args.getBucketName(); String keyName = args.getKeyName(); @@ -115,7 +116,6 @@ public Collection listStatusFSO(OmKeyArgs args, * use the rest of the path to construct prefix path */ - String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); OmBucketInfo omBucketInfo = metadataManager.getBucketTable().get(bucketKey); @@ -130,7 +130,7 @@ public Collection listStatusFSO(OmKeyArgs args, boolean pathNameChanged = false; if (StringUtils.isNotBlank(startKey)) { if (StringUtils.isNotBlank(keyName)) { - if ( !OzoneFSUtils.isImmediateChild(keyName, startKey)) { + if (!listKeysMode && !OzoneFSUtils.isImmediateChild(keyName, startKey)) { if (LOG.isDebugEnabled()) { LOG.debug("StartKey {} is not an immediate child of keyName {}. " + "Returns empty list", startKey, keyName); @@ -151,10 +151,10 @@ public Collection listStatusFSO(OmKeyArgs args, .setSortDatanodesInPipeline(false) .build(); fileStatus = getStatusHelper.apply(startKeyArgs, - null, true); + null, true); } else { fileStatus = - getStatusHelper.apply(args, clientAddress, true); + getStatusHelper.apply(args, clientAddress, listKeysMode); } String dbPrefixKey; From 178a37cb36ead845592583b0bccb341fc3dfad40 Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Thu, 26 May 2022 00:11:02 +0530 Subject: [PATCH 10/13] fix more issues --- .../hadoop/ozone/om/TestListStatus.java | 109 +++---- .../hadoop/ozone/om/KeyManagerImpl.java | 8 +- .../ozone/om/OzoneListStatusHelper.java | 275 ++++++++---------- 3 files changed, 164 insertions(+), 228 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java index a372b445ef89..f40532206d0d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java @@ -16,17 +16,12 @@ */ package org.apache.hadoop.ozone.om; -import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.TestDataUtil; -import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; @@ -41,10 +36,11 @@ import java.util.UUID; import java.util.List; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE; +import static org.apache.hadoop.ozone.OzoneConfigKeys. + OZONE_FS_ITERATE_BATCH_SIZE; /** - * Check that list status output is sorted. + * A simple test that asserts that list status output is sorted. */ public class TestListStatus { @@ -53,8 +49,6 @@ public class TestListStatus { private static String clusterId; private static String scmId; private static String omId; - - private static OzoneBucket legacyOzoneBucket; private static OzoneBucket fsoOzoneBucket; @Rule @@ -79,27 +73,13 @@ public static void init() throws Exception { cluster.waitForClusterToBeReady(); // create a volume and a LEGACY bucket - legacyOzoneBucket = TestDataUtil - .createVolumeAndBucket(cluster, BucketLayout.LEGACY); - String volumeName = legacyOzoneBucket.getVolumeName(); - - // create a volume and a FSO bucket - BucketArgs omBucketArgs; - BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setStorageType(StorageType.DISK); - builder.setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED); - omBucketArgs = builder.build(); - OzoneClient client = cluster.getClient(); - OzoneVolume ozoneVolume = client.getObjectStore().getVolume(volumeName); - - String fsoBucketName = "bucket" + RandomStringUtils.randomNumeric(5); - ozoneVolume.createBucket(fsoBucketName, omBucketArgs); - fsoOzoneBucket = ozoneVolume.getBucket(fsoBucketName); + fsoOzoneBucket = TestDataUtil + .createVolumeAndBucket(cluster, BucketLayout.FILE_SYSTEM_OPTIMIZED); // Set the number of keys to be processed during batch operate. conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5); - initFSNameSpace(); + buildNameSpaceTree(fsoOzoneBucket); } @AfterClass @@ -109,48 +89,28 @@ public static void teardownClass() { } } - private static void initFSNameSpace() throws Exception { - /* - Keys Namespace: - "a1" Dir - "a1/a11" Dir - "a1/a12" File - "a1/a13" File - "a2" File - "a3" Dir - "a3/a31" Dir - "a3/a32" File - "a8" File - "a9" Dir - "a10" File - - "b1" File - "b2" File - "b3" File - "b4" File - */ - buildNameSpaceTree(fsoOzoneBucket); - } - @Test public void testSortedListStatus() throws Exception { - // a) test if output is sorted - checkKeyList("", "", 1000, 10); - - // b) number of keys returns is expected - checkKeyList("", "", 2, 2); - - // c) check if full prefix works - checkKeyList("a1", "", 100, 3); - - // d) check if full prefix with numEntries work - checkKeyList("a1", "", 2, 2); - - // e) check if existing start key >>> - checkKeyList("a1", "a1/a12", 100, 2); +// // a) test if output is sorted +// checkKeyList("", "", 1000, 10); +// +// // b) number of keys returns is expected +// checkKeyList("", "", 2, 2); +// +// // c) check if full prefix works +// checkKeyList("a1", "", 100, 3); +// +// // d) check if full prefix with numEntries work +// checkKeyList("a1", "", 2, 2); +// +// // e) check if existing start key >>> +// checkKeyList("a1", "a1/a12", 100, 2); +// +// // f) check with non existing start key>>> +// checkKeyList("", "a7", 100, 6); - // f) check with non existing start key>>> - checkKeyList("", "a7", 100, 6); + // g) check if full prefix with numEntries work + checkKeyList("a1/", "a1/", 100, 2); // TODO: Enable the following test after listKeys changes // // g) check if half prefix works <<<< @@ -170,6 +130,25 @@ private static void createFile(OzoneBucket bucket, String keyName) } private static void buildNameSpaceTree(OzoneBucket ozoneBucket) throws Exception { + /* + FileSystem Namespace: + "a1" Dir + "a1/a11" Dir + "a1/a12" File + "a1/a13" File + "a2" File + "a3" Dir + "a3/a31" Dir + "a3/a32" File + "a8" File + "a9" Dir + "a10" File + + "b1" File + "b2" File + "b3" File + "b4" File + */ ozoneBucket.createDirectory("/a1"); createFile(ozoneBucket, "/a2"); ozoneBucket.createDirectory("/a3"); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 60537583d17e..694549031ab1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1415,7 +1415,7 @@ public void refresh(OmKeyInfo key) throws IOException { refreshPipeline(Arrays.asList(key)); } - private boolean isKeyDeleted(String key, Table keyTable) { + public static boolean isKeyDeleted(String key, Table keyTable) { CacheValue omKeyInfoCacheValue = keyTable.getCacheValue(new CacheKey(key)); return omKeyInfoCacheValue != null @@ -1501,6 +1501,7 @@ public List listStatus(OmKeyArgs args, boolean recursive, return fileStatusList; } + Preconditions.checkArgument(!recursive); boolean useNewIterator = true; if (isBucketFSOptimized(volName, buckName)) { if (useNewIterator) { @@ -1508,10 +1509,9 @@ public List listStatus(OmKeyArgs args, boolean recursive, new OzoneListStatusHelper(metadataManager, scmBlockSize, this::getOzoneFileStatusFSO); Collection statuses = - statusHelper.listStatusFSO(args, recursive, startKey, numEntries, + statusHelper.listStatusFSO(args, startKey, numEntries, clientAddress); - return buildFinalStatusList(statuses, - args, clientAddress); + return buildFinalStatusList(statuses, args, clientAddress); } else { return listStatusFSO(args, recursive, startKey, numEntries, clientAddress); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index 5c019a80e810..06983f876709 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -41,23 +41,21 @@ import java.util.Iterator; import java.util.TreeMap; import java.util.Map; -import java.util.TreeSet; import java.util.PriorityQueue; -import java.util.Set; import java.util.NoSuchElementException; import java.util.Collection; import java.util.Collections; -import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; +import static org.apache.hadoop.ozone.om.lock. + OzoneManagerLock.Resource.BUCKET_LOCK; /** * Helper class for fetching List Status for a path. */ public class OzoneListStatusHelper { - /** - * Interface to get the File Status. + * Interface to get the File Status for a path. */ @FunctionalInterface public interface GetFileStatusHelper { @@ -65,12 +63,15 @@ OzoneFileStatus apply(OmKeyArgs args, String clientAddress, boolean skipFileNotFoundError) throws IOException; } + public interface ClosableIterator extends Iterator, Closeable { + + } + private static final Logger LOG = LoggerFactory.getLogger(OzoneListStatusHelper.class); private final OMMetadataManager metadataManager; private final long scmBlockSize; - private final GetFileStatusHelper getStatusHelper; OzoneListStatusHelper(OMMetadataManager metadataManager, long scmBlockSize, @@ -81,56 +82,33 @@ OzoneFileStatus apply(OmKeyArgs args, String clientAddress, } public Collection listStatusFSO(OmKeyArgs args, - boolean recursive, String startKey, long numEntries, - String clientAddress) + String startKey, long numEntries, String clientAddress) throws IOException { - Preconditions.checkArgument(!recursive); Preconditions.checkNotNull(args, "Key args can not be null"); - if (numEntries <= 0) { - return new ArrayList<>(); - } - boolean listKeysMode = false; final String volumeName = args.getVolumeName(); final String bucketName = args.getBucketName(); String keyName = args.getKeyName(); String prefixKey = keyName; - /** - * a) If the keyname is a file just return one entry - * b) if the keyname is root, then return the value of the bucket - * c) if the keyname is a different bucket than root, - * fetch the direcoty parent id - * - * if the startkey exists - * a) check the start key is a child ot key, else return emptry list - * b) chekc if the start key is a child chil of keynae, - * then reset the key to parent of start key - * c) if start key is non existent then seek to the neatest key - * d) if the keyname is not a dir or a file, it can either be - * invalid name or a prefix path - * in case, this is called as part of listStatus fail as the - * real dir/file should exist - * else, try to find the parent of keyname and use that as the prefix, - * use the rest of the path to construct prefix path - */ - String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); OmBucketInfo omBucketInfo = metadataManager.getBucketTable().get(bucketKey); if (omBucketInfo == null) { if (LOG.isDebugEnabled()) { - LOG.debug("StartKey {} is not an immediate child of keyName {}. " + - "Returns empty list", startKey, keyName); + LOG.debug("Volume:{} Bucket:{} does not exist", + volumeName, bucketName); } return new ArrayList<>(); } - boolean pathNameChanged = false; + // Determine if the prefixKey is determined from the startKey + // if the keyName is null if (StringUtils.isNotBlank(startKey)) { if (StringUtils.isNotBlank(keyName)) { - if (!listKeysMode && !OzoneFSUtils.isImmediateChild(keyName, startKey)) { + if (!listKeysMode && + !OzoneFSUtils.isImmediateChild(keyName, startKey)) { if (LOG.isDebugEnabled()) { LOG.debug("StartKey {} is not an immediate child of keyName {}. " + "Returns empty list", startKey, keyName); @@ -140,36 +118,35 @@ public Collection listStatusFSO(OmKeyArgs args, } else { keyName = OzoneFSUtils.getParentDir(startKey); prefixKey = keyName; - pathNameChanged = true; + args = args.toBuilder() + .setKeyName(keyName) + .setSortDatanodesInPipeline(false) + .build(); } } - OzoneFileStatus fileStatus = null; - if (pathNameChanged) { - OmKeyArgs startKeyArgs = args.toBuilder() - .setKeyName(keyName) - .setSortDatanodesInPipeline(false) - .build(); - fileStatus = getStatusHelper.apply(startKeyArgs, - null, true); - } else { - fileStatus = - getStatusHelper.apply(args, clientAddress, listKeysMode); - } + OzoneFileStatus fileStatus = + getStatusHelper.apply(args, clientAddress, listKeysMode); String dbPrefixKey; if (fileStatus == null) { + // if the file status is null, prefix is a not a valid filesystem path + // this should only work in list keys mode. + // fetch the db key based on the prefix path. dbPrefixKey = getDbKey(keyName, args, omBucketInfo); prefixKey = OzoneFSUtils.getParentDir(keyName); } else { + // If the keyname is a file just return one entry if (fileStatus.isFile()) { return Collections.singletonList(fileStatus); } + // fetch the db key based on parent prefix id. long id = getId(fileStatus, omBucketInfo); dbPrefixKey = metadataManager.getOzonePathKey(id, ""); } + // Determine startKeyPrefix for DB iteration String startKeyPrefix = Strings.isNullOrEmpty(startKey) ? "" : getDbKey(startKey, args, omBucketInfo); @@ -177,16 +154,16 @@ public Collection listStatusFSO(OmKeyArgs args, TreeMap map = new TreeMap<>(); BucketLayout bucketLayout = omBucketInfo.getBucketLayout(); - try (MinHeapIterator heapIterator = - new MinHeapIterator(metadataManager, dbPrefixKey, bucketLayout, - startKeyPrefix, volumeName, bucketName)) { + + // fetch the sorted output using a min heap iterator where + // every remove from the heap will give the smallest entry. + try(MinHeapIterator heapIterator = new MinHeapIterator(metadataManager, + dbPrefixKey, bucketLayout, startKeyPrefix, volumeName, bucketName)) { while (map.size() < numEntries && heapIterator.hasNext()) { HeapEntry entry = heapIterator.next(); OzoneFileStatus status = entry.getStatus(prefixKey, - scmBlockSize, volumeName, bucketName, args); - LOG.info("returning status:{} keyname:{} startkey:{} numEntries:{}", - status, prefixKey, startKey, numEntries); + scmBlockSize, volumeName, bucketName); map.put(entry.key, status); } } @@ -194,12 +171,13 @@ public Collection listStatusFSO(OmKeyArgs args, return map.values(); } - private String getDbKey(String key, OmKeyArgs args, OmBucketInfo omBucketInfo) throws IOException { long startKeyParentId; String parent = OzoneFSUtils.getParentDir(key); + // the keyname is not a valid filesystem path. + // determine the parent prefix by fetching the OmKeyArgs startKeyArgs = args.toBuilder() .setKeyName(parent) .setSortDatanodesInPipeline(false) @@ -216,14 +194,13 @@ private long getId(OzoneFileStatus fileStatus, OmBucketInfo omBucketInfo) { if (fileStatus.getKeyInfo() != null) { return fileStatus.getKeyInfo().getObjectID(); } else { - // assert root is null // list root directory. return omBucketInfo.getObjectID(); } } /** - * Enum of types of entries. + * Enum of types of entries in the heap */ public enum EntryType { DIR_CACHE, @@ -246,16 +223,17 @@ public boolean isDir() { } /** - * Entry which is added to heap. - * @param + * Entry to be added to the heap */ - private static class HeapEntry - implements Comparable { + private static class HeapEntry implements Comparable { private final EntryType entryType; private final String key; - private final T value; + private final Object value; - HeapEntry(EntryType entryType, String key, T value) { + HeapEntry(EntryType entryType, String key, Object value) { + Preconditions.checkArgument( + value instanceof OmDirectoryInfo || + value instanceof OmKeyInfo); this.entryType = entryType; this.key = key; this.value = value; @@ -283,21 +261,18 @@ public int hashCode() { return key.hashCode(); } - public String getKey() { - return key; - } - public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, - String volumeName, String bucketName, - OmKeyArgs args) { + String volumeName, String bucketName) { OmKeyInfo keyInfo; if (entryType.isDir()) { + Preconditions.checkArgument(value instanceof OmDirectoryInfo); OmDirectoryInfo dirInfo = (OmDirectoryInfo) value; String dirName = OMFileRequest.getAbsolutePath(prefixPath, dirInfo.getName()); keyInfo = OMFileRequest.getOmKeyInfo(volumeName, bucketName, dirInfo, dirName); } else { + Preconditions.checkArgument(value instanceof OmKeyInfo); keyInfo = (OmKeyInfo) value; keyInfo.setFileName(keyInfo.getKeyName()); String fullKeyPath = OMFileRequest.getAbsolutePath(prefixPath, @@ -309,36 +284,25 @@ public OzoneFileStatus getStatus(String prefixPath, long scmBlockSize, } /** - * Iterator class for Ozone keys. + * Iterator for DB entries in a Dir and File Table. */ - public interface OzoneKeyIterator extends - Iterator>, Closeable { - } - - /** - * Raw iterator over db tables. - * @param - */ - private static class RawIter - implements OzoneKeyIterator { - + private static class RawIter implements ClosableIterator { private final EntryType iterType; private final String prefixKey; - private final TableIterator> tableIterator; - private final Set cacheDeletedKeySet; + ? extends Table.KeyValue> tableIterator; + + private final Table table; private HeapEntry currentKey; - RawIter(EntryType iterType, TableIterator> tableIterator, - String prefixKey, String startKey, - Set cacheDeletedKeySet) throws IOException { + RawIter(EntryType iterType, Table table, + String prefixKey, String startKey) throws IOException { this.iterType = iterType; - this.tableIterator = tableIterator; + this.table = table; + this.tableIterator = table.iterator(); this.prefixKey = prefixKey; - this.cacheDeletedKeySet = cacheDeletedKeySet; this.currentKey = null; + if (!StringUtils.isBlank(prefixKey)) { tableIterator.seek(prefixKey); } @@ -352,13 +316,15 @@ private static class RawIter private void getNextKey() throws IOException { while (tableIterator.hasNext() && currentKey == null) { - Table.KeyValue entry = tableIterator.next(); + Table.KeyValue entry = tableIterator.next(); String entryKey = entry.getKey(); if (entryKey.startsWith(prefixKey)) { - if (!cacheDeletedKeySet.contains(entryKey)) { + if (!KeyManagerImpl.isKeyDeleted(entryKey, table)) { currentKey = new HeapEntry(iterType, entryKey, entry.getValue()); } } else { + // if the prefix key does not match, then break + // as the iterator is beyond the prefix. break; } } @@ -385,35 +351,27 @@ public HeapEntry next() { return ret; } - public void close() { - + public void close() throws IOException { + tableIterator.close(); } } /** - * Iterator for Cache for the database. - * @param + * Iterator for Cache entries in a Dir and File Table. */ - private static class CacheIter< T extends WithParentObjectId> - implements OzoneKeyIterator { - private final Set cacheDeletedKeySet; - private final Map cacheKeyMap; - - private final Iterator> + private static class CacheIter + implements ClosableIterator { + private final Map cacheKeyMap; + private final Iterator> cacheCreatedKeyIter; - - private final Iterator, CacheValue>> + private final Iterator, CacheValue>> cacheIter; - private final String prefixKey; private final String startKey; - - private final EntryType entryType; CacheIter(EntryType entryType, Iterator, - CacheValue>> cacheIter, String startKey, String prefixKey) { - this.cacheDeletedKeySet = new TreeSet<>(); + CacheValue>> cacheIter, String startKey, String prefixKey) { this.cacheKeyMap = new TreeMap<>(); this.cacheIter = cacheIter; @@ -428,13 +386,12 @@ private static class CacheIter< T extends WithParentObjectId> private void getCacheValues() { while (cacheIter.hasNext()) { - Map.Entry, CacheValue> entry = + Map.Entry, CacheValue> entry = cacheIter.next(); String cacheKey = entry.getKey().getCacheKey(); - T cacheOmInfo = entry.getValue().getCacheValue(); + Value cacheOmInfo = entry.getValue().getCacheValue(); // cacheOmKeyInfo is null if an entry is deleted in cache if (cacheOmInfo == null) { - cacheDeletedKeySet.add(cacheKey); continue; } @@ -462,36 +419,38 @@ public boolean hasNext() { } public HeapEntry next() { - Map.Entry entry = cacheCreatedKeyIter.next(); + Map.Entry entry = cacheCreatedKeyIter.next(); return new HeapEntry(entryType, entry.getKey(), entry.getValue()); } public void close() { - - } - - public Set getDeletedKeySet() { - return cacheDeletedKeySet; + // Nothing to close here } } /** - * Implement a min heap iterator to find the smaller - * lexicographically sorted string. + * Implement lexicographical sorting of the file status by sorting file status + * across multiple lists. Each of these lists are sorted internally. + * + * This class implements sorted output by implementing a min heap based + * iterator where the initial element from each of sorted list is inserted. + * + * The least entry is removed and the next entry from the same list from + * which the entry is removed is added into the list. + * + * For example + * RawDir - a1, a3, a5, a7 + * RawFile - a2, a4, a6, a8 + * + * Min Heap is initially composed of {(a1, RawDir), (a2, RawFile)} + * THe least element is removed i.e a1 and then next entry from RawDir + * is inserted into minheap resulting in {(a2, RawFile), (a3, RawDir)} + * + * This process is repeated till both the lists are exhausted. */ - private static class MinHeapIterator implements - Iterator>, Closeable { - - private final PriorityQueue> minHeap = new PriorityQueue<>(); - private final ArrayList>> iterators = new ArrayList<>(); - - private final RawIter rawDirIter; - private final RawIter rawFileIter; - - private final CacheIter cacheFileIter; - private final CacheIter cacheDirIter; + private static class MinHeapIterator implements ClosableIterator { + private final PriorityQueue minHeap = new PriorityQueue<>(); + private final ArrayList iterators = new ArrayList<>(); MinHeapIterator(OMMetadataManager omMetadataManager, String prefixKey, BucketLayout bucketLayout, String startKey, @@ -499,54 +458,49 @@ private static class MinHeapIterator implements omMetadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, bucketName); - cacheDirIter = + + // Initialize all the iterators + iterators.add(EntryType.DIR_CACHE.ordinal(), new CacheIter<>(EntryType.DIR_CACHE, omMetadataManager.getDirectoryTable().cacheIterator(), - startKey, prefixKey); + startKey, prefixKey)); - cacheFileIter = + iterators.add(EntryType.FILE_CACHE.ordinal(), new CacheIter<>(EntryType.FILE_CACHE, omMetadataManager.getKeyTable(bucketLayout).cacheIterator(), - startKey, prefixKey); + startKey, prefixKey)); - rawDirIter = + iterators.add(EntryType.RAW_DIR_DB.ordinal(), new RawIter<>(EntryType.RAW_DIR_DB, - omMetadataManager.getDirectoryTable().iterator(), - prefixKey, startKey, cacheDirIter.getDeletedKeySet()); + omMetadataManager.getDirectoryTable(), + prefixKey, startKey)); - rawFileIter = + iterators.add(EntryType.RAW_FILE_DB.ordinal(), new RawIter<>(EntryType.RAW_FILE_DB, - omMetadataManager.getKeyTable(bucketLayout).iterator(), - prefixKey, startKey, cacheFileIter.getDeletedKeySet()); + omMetadataManager.getKeyTable(bucketLayout), + prefixKey, startKey)); omMetadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, bucketName); - iterators.add(EntryType.DIR_CACHE.ordinal(), cacheDirIter); - iterators.add(EntryType.FILE_CACHE.ordinal(), cacheFileIter); - iterators.add(EntryType.RAW_FILE_DB.ordinal(), rawFileIter); - iterators.add(EntryType.RAW_DIR_DB.ordinal(), rawDirIter); - insertFirstElement(); - - } - - public void insertFirstElement() { - for (Iterator> iter : - iterators) { + // Insert the element from each of the iterator + for (Iterator iter : iterators) { if (iter.hasNext()) { minHeap.add(iter.next()); } } } + public boolean hasNext() { return !minHeap.isEmpty(); } - public HeapEntry next() { - HeapEntry heapEntry = minHeap.remove(); - Iterator> iter = - iterators.get(heapEntry.entryType.ordinal()); + public HeapEntry next() { + HeapEntry heapEntry = minHeap.remove(); + // remove the least element and + // reinsert the next element from the same iterator + Iterator iter = iterators.get(heapEntry.entryType.ordinal()); if (iter.hasNext()) { minHeap.add(iter.next()); } @@ -555,6 +509,9 @@ public HeapEntry next() { } public void close() throws IOException { + for (ClosableIterator iterator : iterators) { + iterator.close(); + } } } } From 687d91f586efa05b8f5563e76a59bf0186efb82a Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Thu, 26 May 2022 00:11:58 +0530 Subject: [PATCH 11/13] fix more issues 2 --- .../hadoop/ozone/om/TestListStatus.java | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java index f40532206d0d..2fe2d687045b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListStatus.java @@ -91,26 +91,23 @@ public static void teardownClass() { @Test public void testSortedListStatus() throws Exception { -// // a) test if output is sorted -// checkKeyList("", "", 1000, 10); -// -// // b) number of keys returns is expected -// checkKeyList("", "", 2, 2); -// -// // c) check if full prefix works -// checkKeyList("a1", "", 100, 3); -// -// // d) check if full prefix with numEntries work -// checkKeyList("a1", "", 2, 2); -// -// // e) check if existing start key >>> -// checkKeyList("a1", "a1/a12", 100, 2); -// -// // f) check with non existing start key>>> -// checkKeyList("", "a7", 100, 6); + // a) test if output is sorted + checkKeyList("", "", 1000, 10); + + // b) number of keys returns is expected + checkKeyList("", "", 2, 2); + + // c) check if full prefix works + checkKeyList("a1", "", 100, 3); + + // d) check if full prefix with numEntries work + checkKeyList("a1", "", 2, 2); + + // e) check if existing start key >>> + checkKeyList("a1", "a1/a12", 100, 2); - // g) check if full prefix with numEntries work - checkKeyList("a1/", "a1/", 100, 2); + // f) check with non existing start key>>> + checkKeyList("", "a7", 100, 6); // TODO: Enable the following test after listKeys changes // // g) check if half prefix works <<<< From ec5919b6153fb08b2bbe5992808139b95cf2e250 Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Thu, 26 May 2022 00:18:23 +0530 Subject: [PATCH 12/13] fix more issues 2 --- .../apache/hadoop/ozone/om/OzoneListStatusHelper.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index 06983f876709..c11394b8ffe0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -63,6 +63,9 @@ OzoneFileStatus apply(OmKeyArgs args, String clientAddress, boolean skipFileNotFoundError) throws IOException; } + /** + * Interface for iteration of Heap Entries. + */ public interface ClosableIterator extends Iterator, Closeable { } @@ -157,7 +160,7 @@ public Collection listStatusFSO(OmKeyArgs args, // fetch the sorted output using a min heap iterator where // every remove from the heap will give the smallest entry. - try(MinHeapIterator heapIterator = new MinHeapIterator(metadataManager, + try (MinHeapIterator heapIterator = new MinHeapIterator(metadataManager, dbPrefixKey, bucketLayout, startKeyPrefix, volumeName, bucketName)) { while (map.size() < numEntries && heapIterator.hasNext()) { @@ -200,7 +203,7 @@ private long getId(OzoneFileStatus fileStatus, OmBucketInfo omBucketInfo) { } /** - * Enum of types of entries in the heap + * Enum of types of entries in the heap. */ public enum EntryType { DIR_CACHE, @@ -223,7 +226,7 @@ public boolean isDir() { } /** - * Entry to be added to the heap + * Entry to be added to the heap. */ private static class HeapEntry implements Comparable { private final EntryType entryType; From a71a7be855ba7ec4ab8ee4e325caaeee37cf2fe0 Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Thu, 26 May 2022 06:47:59 +0530 Subject: [PATCH 13/13] fix issues --- .../main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java | 2 +- .../org/apache/hadoop/ozone/om/OzoneListStatusHelper.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 694549031ab1..0dc8d68cdd1f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -1501,9 +1501,9 @@ public List listStatus(OmKeyArgs args, boolean recursive, return fileStatusList; } - Preconditions.checkArgument(!recursive); boolean useNewIterator = true; if (isBucketFSOptimized(volName, buckName)) { + Preconditions.checkArgument(!recursive); if (useNewIterator) { OzoneListStatusHelper statusHelper = new OzoneListStatusHelper(metadataManager, scmBlockSize, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java index c11394b8ffe0..a6dbcd042838 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneListStatusHelper.java @@ -208,8 +208,8 @@ private long getId(OzoneFileStatus fileStatus, OmBucketInfo omBucketInfo) { public enum EntryType { DIR_CACHE, FILE_CACHE, - RAW_FILE_DB, - RAW_DIR_DB; + RAW_DIR_DB, + RAW_FILE_DB; public boolean isDir() { switch (this) {