diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java index 8c89f7c4e2d3..492173b71cf9 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java @@ -1336,7 +1336,7 @@ public void testRefreshPipeline() throws Exception { omKeyInfo.appendNewBlocks(omKeyLocationInfoList, false); KeyManagerImpl keyManagerImpl = - new KeyManagerImpl(ozoneManager, scmClientMock, conf, "om1", metrics); + new KeyManagerImpl(ozoneManager, scmClientMock, conf, metrics); keyManagerImpl.refresh(omKeyInfo); @@ -1377,7 +1377,7 @@ public void testRefreshPipelineException() throws Exception { omKeyInfo.appendNewBlocks(omKeyLocationInfoList, false); KeyManagerImpl keyManagerImpl = - new KeyManagerImpl(ozoneManager, scmClientMock, conf, "om1", metrics); + new KeyManagerImpl(ozoneManager, scmClientMock, conf, metrics); try { keyManagerImpl.refresh(omKeyInfo); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 3de705328716..4ea8c52bf430 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -37,7 +37,6 @@ import java.util.Set; import java.util.Stack; import java.util.TreeMap; -import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -105,19 +104,14 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED_DEFAULT; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.BlockTokenSecretProto.AccessModeProto.READ; -import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT; -import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_TRASH_KEYS_MAX; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_TRASH_KEYS_MAX_DEFAULT; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_BLOCKS_MAX_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT; -import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL_DEFAULT; @@ -127,7 +121,6 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_TIMEOUT_DEFAULT; import static org.apache.hadoop.ozone.om.OzoneManagerUtils.getBucketLayout; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.DIRECTORY_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INTERNAL_ERROR; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KMS_PROVIDER; @@ -155,19 +148,13 @@ public class KeyManagerImpl implements KeyManager { private final ScmClient scmClient; private final OMMetadataManager metadataManager; private final long scmBlockSize; - private final boolean useRatis; - - private final int preallocateBlocksMax; private final int listTrashKeysMax; - private final String omId; private final OzoneBlockTokenSecretManager secretManager; private final boolean grpcBlockTokenEnabled; private BackgroundService keyDeletingService; private final KeyProviderCryptoExtension kmsProvider; - private final PrefixManager prefixManager; - private final boolean enableFileSystemPaths; private BackgroundService dirDeletingService; private final OMPerformanceMetrics metrics; @@ -176,11 +163,11 @@ public class KeyManagerImpl implements KeyManager { @VisibleForTesting public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient, - OMMetadataManager metadataManager, OzoneConfiguration conf, String omId, + OMMetadataManager metadataManager, OzoneConfiguration conf, OzoneBlockTokenSecretManager secretManager, OMPerformanceMetrics metrics) { this(null, new ScmClient(scmBlockClient, null, conf), metadataManager, - conf, omId, secretManager, null, null, metrics); + conf, secretManager, null, metrics); } @VisibleForTesting @@ -190,31 +177,24 @@ public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient, OzoneBlockTokenSecretManager secretManager, OMPerformanceMetrics metrics) { this(null, new ScmClient(scmBlockClient, scmContainerClient, conf), - metadataManager, conf, omId, secretManager, null, null, + metadataManager, conf, secretManager, null, metrics); } public KeyManagerImpl(OzoneManager om, ScmClient scmClient, - OzoneConfiguration conf, String omId, OMPerformanceMetrics metrics) { - this (om, scmClient, om.getMetadataManager(), conf, omId, - om.getBlockTokenMgr(), om.getKmsProvider(), om.getPrefixManager(), - metrics); + OzoneConfiguration conf, OMPerformanceMetrics metrics) { + this (om, scmClient, om.getMetadataManager(), conf, + om.getBlockTokenMgr(), om.getKmsProvider(), metrics); } - @SuppressWarnings("parameternumber") public KeyManagerImpl(OzoneManager om, ScmClient scmClient, - OMMetadataManager metadataManager, OzoneConfiguration conf, String omId, + OMMetadataManager metadataManager, OzoneConfiguration conf, OzoneBlockTokenSecretManager secretManager, - KeyProviderCryptoExtension kmsProvider, PrefixManager prefixManager, + KeyProviderCryptoExtension kmsProvider, OMPerformanceMetrics metrics) { this.scmBlockSize = (long) conf .getStorageSize(OZONE_SCM_BLOCK_SIZE, OZONE_SCM_BLOCK_SIZE_DEFAULT, StorageUnit.BYTES); - this.useRatis = conf.getBoolean(DFS_CONTAINER_RATIS_ENABLED_KEY, - DFS_CONTAINER_RATIS_ENABLED_DEFAULT); - this.preallocateBlocksMax = conf.getInt( - OZONE_KEY_PREALLOCATION_BLOCKS_MAX, - OZONE_KEY_PREALLOCATION_BLOCKS_MAX_DEFAULT); this.grpcBlockTokenEnabled = conf.getBoolean( HDDS_BLOCK_TOKEN_ENABLED, HDDS_BLOCK_TOKEN_ENABLED_DEFAULT); @@ -226,14 +206,11 @@ public KeyManagerImpl(OzoneManager om, ScmClient scmClient, OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT); this.ozoneManager = om; - this.omId = omId; this.scmClient = scmClient; this.metadataManager = metadataManager; - this.prefixManager = prefixManager; this.secretManager = secretManager; this.kmsProvider = kmsProvider; this.metrics = metrics; - } @Override @@ -309,26 +286,6 @@ private OmBucketInfo getBucketInfo(String volumeName, String bucketName) return metadataManager.getBucketTable().get(bucketKey); } - /** - * Check S3 bucket exists or not. - * @param volumeName - * @param bucketName - * @throws IOException - */ - private OmBucketInfo validateS3Bucket(String volumeName, String bucketName) - throws IOException { - - String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); - OmBucketInfo omBucketInfo = metadataManager.getBucketTable(). - get(bucketKey); - //Check if bucket already exists - if (omBucketInfo == null) { - LOG.error("bucket not found: {}/{} ", volumeName, bucketName); - throw new OMException("Bucket not found", - BUCKET_NOT_FOUND); - } - return omBucketInfo; - } /* Optimize ugi lookup for RPC operations to avoid a trip through * UGI.getCurrentUser which is synch'ed. */ @@ -560,16 +517,6 @@ protected Map refreshPipeline( } } - private boolean isKeyEmpty(OmKeyInfo keyInfo) { - for (OmKeyLocationInfoGroup keyLocationList : keyInfo - .getKeyLocationVersions()) { - if (keyLocationList.getLocationListCount() != 0) { - return false; - } - } - return true; - } - @Override public List listKeys(String volumeName, String bucketName, String startKey, String keyPrefix, @@ -784,8 +731,6 @@ public OmMultipartUploadListParts listParts(String volumeName, "ReplicationConfig can't be identified"); if (partKeyInfoMapIterator.hasNext()) { - Map.Entry partKeyInfoEntry = - partKeyInfoMapIterator.next(); isTruncated = true; } else { isTruncated = false; @@ -1551,21 +1496,15 @@ public List listStatus(OmKeyArgs args, boolean recursive, return fileStatusList; } - boolean useNewIterator = true; if (isBucketFSOptimized(volName, buckName)) { Preconditions.checkArgument(!recursive); - if (useNewIterator) { - OzoneListStatusHelper statusHelper = - new OzoneListStatusHelper(metadataManager, scmBlockSize, - this::getOzoneFileStatusFSO); - Collection statuses = - statusHelper.listStatusFSO(args, startKey, numEntries, - clientAddress, allowPartialPrefixes); - return buildFinalStatusList(statuses, args, clientAddress); - } else { - return listStatusFSO(args, recursive, startKey, numEntries, - clientAddress); - } + OzoneListStatusHelper statusHelper = + new OzoneListStatusHelper(metadataManager, scmBlockSize, + this::getOzoneFileStatusFSO); + Collection statuses = + statusHelper.listStatusFSO(args, startKey, numEntries, + clientAddress, allowPartialPrefixes); + return buildFinalStatusList(statuses, args, clientAddress); } String volumeName = args.getVolumeName(); @@ -1726,270 +1665,6 @@ private void findKeyInDbWithIterator(boolean recursive, String startKey, } } - @SuppressWarnings("methodlength") - public List listStatusFSO(OmKeyArgs args, boolean recursive, - String startKey, long numEntries, String clientAddress) - throws IOException { - Preconditions.checkNotNull(args, "Key args can not be null"); - - // unsorted OMKeyInfo list contains combine results from TableCache and DB. - - if (numEntries <= 0) { - return new ArrayList<>(); - } - - /** - * A map sorted by OmKey to combine results from TableCache and DB for - * each entity - Dir & File. - * - * Two separate maps are required because the order of seek -> (1)Seek - * files in fileTable (2)Seek dirs in dirTable. - * - * StartKey should be added to the final listStatuses, so if we combine - * files and dirs into a single map then directory with lower precedence - * will appear at the top of the list even if the startKey is given as - * fileName. - * - * For example, startKey="a/file1". As per the seek order, first fetches - * all the files and then it will start seeking all the directories. - * Assume a directory name exists "a/b". With one map, the sorted list will - * be ["a/b", "a/file1"]. But the expected list is: ["a/file1", "a/b"], - * startKey element should always be at the top of the listStatuses. - */ - TreeMap cacheFileMap = new TreeMap<>(); - TreeMap cacheDirMap = new TreeMap<>(); - - final String volumeName = args.getVolumeName(); - final String bucketName = args.getBucketName(); - final String keyName = args.getKeyName(); - final long volumeId = metadataManager.getVolumeId(volumeName); - final long bucketId = metadataManager.getBucketId(volumeName, bucketName); - String seekFileInDB; - String seekDirInDB; - long prefixKeyInDB; - String prefixPath = keyName; - int countEntries = 0; - - // TODO: recursive flag=true will be handled in HDDS-4360 jira. - - - Set deletedKeySet = new TreeSet<>(); - TreeMap tempCacheDirMap = new TreeMap<>(); - - TableIterator> - iterator = null; - - try { - if (Strings.isNullOrEmpty(startKey)) { - OzoneFileStatus fileStatus = getFileStatus(args, clientAddress); - if (fileStatus.isFile()) { - return Collections.singletonList(fileStatus); - } - - // Not required to search in DeletedTable because all the deleted - // keys will be marked directly in dirTable or in keyTable by - // breaking the pointer to its sub-dirs and sub-files. So, there is no - // issue of inconsistency. - - /* - * keyName is a directory. - * Say, "/a" is the dir name and its objectID is 1024, then seek - * will be doing with "1024/" to get all immediate descendants. - */ - if (fileStatus.getKeyInfo() != null) { - prefixKeyInDB = fileStatus.getKeyInfo().getObjectID(); - } else { - // list root directory. - prefixKeyInDB = bucketId; - } - seekFileInDB = metadataManager.getOzonePathKey( - volumeId, bucketId, prefixKeyInDB, ""); - seekDirInDB = metadataManager.getOzonePathKey( - volumeId, bucketId, prefixKeyInDB, ""); - - // Order of seek -> - // (1)Seek files in fileTable - // (2)Seek dirs in dirTable - - // First under lock obtain both entries from dir/file cache and generate - // entries marked for delete. - metadataManager.getLock() - .acquireReadLock(BUCKET_LOCK, volumeName, bucketName); - try { - BucketLayout bucketLayout = getBucketLayout( - metadataManager, volumeName, bucketName); - iterator = metadataManager.getKeyTable(bucketLayout).iterator(); - countEntries = - getFilesAndDirsFromCacheWithBucket(volumeName, bucketName, - cacheFileMap, tempCacheDirMap, deletedKeySet, prefixKeyInDB, - seekFileInDB, seekDirInDB, prefixPath, startKey, countEntries, - numEntries); - - } finally { - metadataManager.getLock() - .releaseReadLock(BUCKET_LOCK, volumeName, bucketName); - } - countEntries = - getFilesFromDirectory(cacheFileMap, seekFileInDB, prefixPath, - prefixKeyInDB, countEntries, numEntries, deletedKeySet, - iterator); - - } else { - /* - * startKey will be used in iterator seek and sets the beginning point - * for key traversal. - * keyName will be used as parentID where the user has requested to - * list the keys from. - * - * When recursive flag=false, parentID won't change between two pages. - * For example: OM has a namespace like, - * /a/1...1M files and /a/b/1...1M files. - * /a/1...1M directories and /a/b/1...1M directories. - * Listing "/a", will always have the parentID as "a" irrespective of - * the startKey value. - */ - - // Check startKey is an immediate child of keyName. For example, - // keyName=/a/ and expected startKey=/a/b. startKey can't be /xyz/b. - if (StringUtils.isNotBlank(keyName) && !OzoneFSUtils - .isImmediateChild(keyName, startKey)) { - if (LOG.isDebugEnabled()) { - LOG.debug("StartKey {} is not an immediate child of keyName {}. " - + "Returns empty list", startKey, keyName); - } - return Collections.emptyList(); - } - - // assign startKeyPath if prefixPath is empty string. - if (StringUtils.isBlank(prefixPath)) { - prefixPath = OzoneFSUtils.getParentDir(startKey); - } - - OmKeyArgs startKeyArgs = args.toBuilder() - .setKeyName(startKey) - .setSortDatanodesInPipeline(false) - .build(); - OzoneFileStatus fileStatusInfo = getOzoneFileStatusFSO(startKeyArgs, - null, true); - - if (fileStatusInfo != null) { - prefixKeyInDB = fileStatusInfo.getKeyInfo().getParentObjectID(); - - if (fileStatusInfo.isDirectory()) { - seekDirInDB = metadataManager.getOzonePathKey( - volumeId, bucketId, prefixKeyInDB, - fileStatusInfo.getKeyInfo().getFileName()); - - // Order of seek -> (1) Seek dirs only in dirTable. In OM, always - // the order of search is, first seek into fileTable and then - // dirTable. So, its not required to search again in the fileTable. - - // Seek the given key in dirTable. - metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, - bucketName); - try { - listStatusFindDirsInTableCache(tempCacheDirMap, - metadataManager.getDirectoryTable(), - prefixKeyInDB, seekDirInDB, prefixPath, startKey, volumeName, - bucketName, countEntries, numEntries, deletedKeySet); - } finally { - metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, - bucketName); - } - - } else { - seekFileInDB = metadataManager.getOzonePathKey( - volumeId, bucketId, prefixKeyInDB, - fileStatusInfo.getKeyInfo().getFileName()); - // begins from the first sub-dir under the parent dir - seekDirInDB = metadataManager.getOzonePathKey( - volumeId, bucketId, prefixKeyInDB, ""); - - // First under lock obtain both entries from dir/file cache and - // generate entries marked for delete. - metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, - bucketName); - try { - BucketLayout bucketLayout = - getBucketLayout(metadataManager, volumeName, bucketName); - iterator = metadataManager.getKeyTable(bucketLayout) - .iterator(); - countEntries = getFilesAndDirsFromCacheWithBucket(volumeName, - bucketName, cacheFileMap, tempCacheDirMap, deletedKeySet, - prefixKeyInDB, seekFileInDB, seekDirInDB, prefixPath, - startKey, countEntries, numEntries); - } finally { - metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName, - bucketName); - } - - // 1. Seek the given key in key table. - countEntries = getFilesFromDirectory(cacheFileMap, seekFileInDB, - prefixPath, prefixKeyInDB, countEntries, numEntries, - deletedKeySet, iterator); - } - } else { - // TODO: HDDS-4364: startKey can be a non-existed key - if (LOG.isDebugEnabled()) { - LOG.debug("StartKey {} is a non-existed key and returning empty " - + "list", startKey); - } - return Collections.emptyList(); - } - } - } finally { - if (iterator != null) { - iterator.close(); - } - } - - // Now first add all entries in the dir cache, if numEntries required - // is less than countEntries. - for (Map.Entry dirEntry : - tempCacheDirMap.entrySet()) { - if (countEntries < numEntries) { - cacheDirMap.put(dirEntry.getKey(), dirEntry.getValue()); - countEntries++; - } - } - - // 2. Seek the given key in dir table. - if (countEntries < numEntries) { - getDirectories(cacheDirMap, seekDirInDB, prefixPath, - prefixKeyInDB, countEntries, numEntries, recursive, - volumeName, bucketName, deletedKeySet); - } - - - return buildFinalStatusList(cacheFileMap, cacheDirMap, args, clientAddress); - - } - - /** - * Build final OzoneFileStatus list to be returned to client. - * @throws IOException - */ - private List buildFinalStatusList( - Map cacheFileMap, - Map cacheDirMap, OmKeyArgs omKeyArgs, - String clientAddress) - throws IOException { - List keyInfoList = new ArrayList<>(); - List fileStatusFinalList = new ArrayList<>(); - - for (OzoneFileStatus fileStatus : cacheFileMap.values()) { - fileStatusFinalList.add(fileStatus); - keyInfoList.add(fileStatus.getKeyInfo()); - } - - for (OzoneFileStatus fileStatus : cacheDirMap.values()) { - fileStatusFinalList.add(fileStatus); - } - - return sortPipelineInfo(fileStatusFinalList, keyInfoList, - omKeyArgs, clientAddress); - } - private List buildFinalStatusList( Collection statusesCollection, OmKeyArgs omKeyArgs, String clientAddress) @@ -2029,255 +1704,6 @@ private List sortPipelineInfo( return fileStatusFinalList; } - - /*** - * Build files, directories and marked for deleted entries from dir/file - * cache. - */ - @SuppressWarnings("parameternumber") - private int getFilesAndDirsFromCacheWithBucket(String volumeName, - String bucketName, Map cacheFileMap, - Map tempCacheDirMap, - Set deletedKeySet, long prefixKeyInDB, - String seekFileInDB, String seekDirInDB, String prefixPath, - String startKey, int countEntries, long numEntries) throws IOException { - - - // First under lock obtain both entries from dir/file cache and generate - // entries marked for delete. - BucketLayout bucketLayout = getBucketLayout(metadataManager, volumeName, - bucketName); - countEntries = listStatusFindFilesInTableCache(cacheFileMap, metadataManager - .getKeyTable(bucketLayout), - prefixKeyInDB, seekFileInDB, prefixPath, startKey, countEntries, - numEntries, deletedKeySet); - - // Don't count entries from dir cache, as first we need to return all - // files and then directories. - listStatusFindDirsInTableCache(tempCacheDirMap, - metadataManager.getDirectoryTable(), - prefixKeyInDB, seekDirInDB, prefixPath, startKey, volumeName, - bucketName, countEntries, numEntries, deletedKeySet); - - return countEntries; - } - - @SuppressWarnings("parameternumber") - protected int getDirectories( - TreeMap cacheKeyMap, - String seekDirInDB, String prefixPath, long prefixKeyInDB, - int countEntries, long numEntries, boolean recursive, - String volumeName, String bucketName, Set deletedKeySet) - throws IOException { - - Table dirTable = metadataManager.getDirectoryTable(); - try (TableIterator> - iterator = dirTable.iterator()) { - - return getDirectoriesWithIterator(cacheKeyMap, seekDirInDB, prefixPath, - prefixKeyInDB, countEntries, numEntries, recursive, volumeName, - bucketName, deletedKeySet, iterator); - } - } - - @SuppressWarnings("parameternumber") - private int getDirectoriesWithIterator( - TreeMap cacheKeyMap, String seekDirInDB, - String prefixPath, long prefixKeyInDB, int countEntries, long numEntries, - boolean recursive, String volumeName, String bucketName, - Set deletedKeySet, - TableIterator> iterator) - throws IOException { - iterator.seek(seekDirInDB); - - while (iterator.hasNext() && numEntries - countEntries > 0) { - Table.KeyValue entry = iterator.next(); - OmDirectoryInfo dirInfo = entry.getValue(); - if (deletedKeySet.contains(dirInfo.getPath())) { - iterator.next(); // move to next entry in the table - // entry is actually deleted in cache and can exists in DB - continue; - } - if (!OMFileRequest.isImmediateChild(dirInfo.getParentObjectID(), - prefixKeyInDB)) { - break; - } - - // TODO: recursive list will be handled in HDDS-4360 jira. - if (!recursive) { - String dirName = OMFileRequest.getAbsolutePath(prefixPath, - dirInfo.getName()); - OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo(volumeName, - bucketName, dirInfo, dirName); - cacheKeyMap.put(dirName, new OzoneFileStatus(omKeyInfo, scmBlockSize, - true)); - countEntries++; - } - } - - return countEntries; - } - - @SuppressWarnings("parameternumber") - private int getFilesFromDirectory( - TreeMap cacheKeyMap, - String seekKeyInDB, String prefixKeyPath, long prefixKeyInDB, - int countEntries, long numEntries, Set deletedKeySet, - TableIterator> - iterator) - throws IOException { - iterator.seek(seekKeyInDB); - while (iterator.hasNext() && numEntries - countEntries > 0) { - Table.KeyValue entry = iterator.next(); - OmKeyInfo keyInfo = entry.getValue(); - if (!entry.getKey().startsWith(seekKeyInDB)) { - break; - } - final long volumeId = metadataManager.getVolumeId( - keyInfo.getVolumeName()); - final long bucketId = metadataManager.getBucketId( - keyInfo.getVolumeName(), keyInfo.getBucketName()); - final String keyPath = metadataManager.getOzonePathKey(volumeId, - bucketId, keyInfo.getParentObjectID(), keyInfo.getFileName()); - if (deletedKeySet.contains(keyPath)) { - // move to next entry in the table - // entry is actually deleted in cache and can exist in DB - continue; - } - if (!OMFileRequest.isImmediateChild(keyInfo.getParentObjectID(), - prefixKeyInDB)) { - break; - } - - keyInfo.setFileName(keyInfo.getKeyName()); - String fullKeyPath = OMFileRequest.getAbsolutePath(prefixKeyPath, - keyInfo.getKeyName()); - keyInfo.setKeyName(fullKeyPath); - cacheKeyMap.put(fullKeyPath, - new OzoneFileStatus(keyInfo, scmBlockSize, false)); - countEntries++; - } - return countEntries; - } - - /** - * Helper function for listStatus to find key in FileTableCache. - */ - @SuppressWarnings("parameternumber") - private int listStatusFindFilesInTableCache( - Map cacheKeyMap, Table keyTable, long prefixKeyInDB, String seekKeyInDB, - String prefixKeyPath, String startKey, int countEntries, - long numEntries, Set deletedKeySet) { - - Iterator, CacheValue>> - cacheIter = keyTable.cacheIterator(); - - // TODO: recursive list will be handled in HDDS-4360 jira. - while (cacheIter.hasNext() && numEntries - countEntries > 0) { - Map.Entry, CacheValue> entry = - cacheIter.next(); - String cacheKey = entry.getKey().getCacheKey(); - OmKeyInfo cacheOmKeyInfo = entry.getValue().getCacheValue(); - // cacheOmKeyInfo is null if an entry is deleted in cache - if (cacheOmKeyInfo == null) { - deletedKeySet.add(cacheKey); - continue; - } - - // make OmKeyInfo local copy to reset keyname to "fullKeyPath". - // In DB keyName stores only the leaf node but the list - // returning to the user should have full path. - OmKeyInfo omKeyInfo = cacheOmKeyInfo.copyObject(); - - omKeyInfo.setFileName(omKeyInfo.getKeyName()); - String fullKeyPath = OMFileRequest.getAbsolutePath(prefixKeyPath, - omKeyInfo.getKeyName()); - omKeyInfo.setKeyName(fullKeyPath); - - countEntries = addKeyInfoToFileStatusList(cacheKeyMap, prefixKeyInDB, - seekKeyInDB, startKey, countEntries, cacheKey, omKeyInfo, - false); - } - return countEntries; - } - - /** - * Helper function for listStatus to find key in DirTableCache. - */ - @SuppressWarnings("parameternumber") - private int listStatusFindDirsInTableCache( - Map cacheKeyMap, Table dirTable, long prefixKeyInDB, String seekKeyInDB, - String prefixKeyPath, String startKey, String volumeName, - String bucketName, int countEntries, long numEntries, - Set deletedKeySet) { - - Iterator, CacheValue>> - cacheIter = dirTable.cacheIterator(); - // seekKeyInDB will have two type of values. - // 1. "1024/" -> startKey is null or empty - // 2. "1024/b" -> startKey exists - // TODO: recursive list will be handled in HDDS-4360 jira. - while (cacheIter.hasNext() && numEntries - countEntries > 0) { - Map.Entry, CacheValue> entry = - cacheIter.next(); - String cacheKey = entry.getKey().getCacheKey(); - OmDirectoryInfo cacheOmDirInfo = entry.getValue().getCacheValue(); - // cacheOmKeyInfo is null if an entry is deleted in cache - if (cacheOmDirInfo == null) { - deletedKeySet.add(cacheKey); - continue; - } - String fullDirPath = OMFileRequest.getAbsolutePath(prefixKeyPath, - cacheOmDirInfo.getName()); - OmKeyInfo cacheDirKeyInfo = OMFileRequest.getOmKeyInfo(volumeName, - bucketName, cacheOmDirInfo, fullDirPath); - - countEntries = addKeyInfoToFileStatusList(cacheKeyMap, prefixKeyInDB, - seekKeyInDB, startKey, countEntries, cacheKey, cacheDirKeyInfo, - true); - } - return countEntries; - } - - @SuppressWarnings("parameternumber") - private int addKeyInfoToFileStatusList( - Map cacheKeyMap, - long prefixKeyInDB, String seekKeyInDB, String startKey, - int countEntries, String cacheKey, OmKeyInfo cacheOmKeyInfo, - boolean isDirectory) { - // seekKeyInDB will have two type of values. - // 1. "1024/" -> startKey is null or empty - // 2. "1024/b" -> startKey exists - if (StringUtils.isBlank(startKey)) { - // startKey is null or empty, then the seekKeyInDB="1024/" - if (cacheKey.startsWith(seekKeyInDB)) { - OzoneFileStatus fileStatus = new OzoneFileStatus(cacheOmKeyInfo, - scmBlockSize, isDirectory); - cacheKeyMap.put(cacheOmKeyInfo.getKeyName(), fileStatus); - countEntries++; - } - } else { - // startKey not empty, then the seekKeyInDB="1024/b" and - // seekKeyInDBWithOnlyParentID = "1024/". This is to avoid case of - // parentID with "102444" cache entries. - // Here, it has to list all the keys after "1024/b" and requires >=0 - // string comparison. - String seekKeyInDBWithOnlyParentID = prefixKeyInDB + OM_KEY_PREFIX; - if (cacheKey.startsWith(seekKeyInDBWithOnlyParentID) && - cacheKey.compareTo(seekKeyInDB) >= 0) { - OzoneFileStatus fileStatus = new OzoneFileStatus(cacheOmKeyInfo, - scmBlockSize, isDirectory); - cacheKeyMap.put(cacheOmKeyInfo.getKeyName(), fileStatus); - countEntries++; - } - } - return countEntries; - } - private String getNextGreaterString(String volumeName, String bucketName, String keyPrefix) throws IOException { // Increment the last character of the string and return the new ozone key. @@ -2291,61 +1717,6 @@ private String getNextGreaterString(String volumeName, String bucketName, return metadataManager.getOzoneKey(volumeName, bucketName, nextPrefix); } - /** - * Verify that none of the parent path exists as file in the filesystem. - * - * @param volumeName Volume name - * @param bucketName Bucket name - * @param path Directory path. This is the absolute path of the - * directory for the ozone filesystem. - * @param directoryMustExist throws exception if true and given path does not - * exist as directory - * @return OzoneFileStatus of the first directory found in path in reverse - * order - * @throws OMException if ancestor exists as file in the filesystem - * if directoryMustExist flag is true and parent does - * not exist - * if bucket does not exist - * @throws IOException if there is error in the db - * invalid arguments - */ - private OzoneFileStatus verifyNoFilesInPath(String volumeName, - String bucketName, Path path, boolean directoryMustExist) - throws IOException { - OmKeyArgs.Builder argsBuilder = new OmKeyArgs.Builder() - .setVolumeName(volumeName) - .setBucketName(bucketName); - while (path != null) { - String keyName = path.toString(); - try { - OzoneFileStatus fileStatus = - getFileStatus(argsBuilder.setKeyName(keyName).build()); - if (fileStatus.isFile()) { - LOG.error("Unable to create directory (File already exists): " - + "volume: {} bucket: {} key: {}", volumeName, bucketName, - keyName); - throw new OMException( - "Unable to create directory at : volume: " + volumeName - + "bucket: " + bucketName + "key: " + keyName, - ResultCodes.FILE_ALREADY_EXISTS); - } else if (fileStatus.isDirectory()) { - return fileStatus; - } - } catch (OMException ex) { - if (ex.getResult() != FILE_NOT_FOUND) { - throw ex; - } else if (ex.getResult() == FILE_NOT_FOUND) { - if (directoryMustExist) { - throw new OMException("Parent directory does not exist", - ex.getCause(), DIRECTORY_NOT_FOUND); - } - } - } - path = path.getParent(); - } - return null; - } - private FileEncryptionInfo getFileEncryptionInfo(OmBucketInfo bucketInfo) throws IOException { FileEncryptionInfo encInfo = null; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 388fc2910fb1..8d6ae64ddfed 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -724,7 +724,7 @@ private void instantiateServices(boolean withNewSnapshot) throws IOException { prefixManager = new PrefixManagerImpl(metadataManager, isRatisEnabled); perfMetrics = OMPerformanceMetrics.register(); keyManager = new KeyManagerImpl(this, scmClient, configuration, - omStorage.getOmId(), perfMetrics); + perfMetrics); if (withNewSnapshot) { Integer layoutVersionInDB = getLayoutVersionInDB(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java index a4906d8f2e79..2a6f17ad61f4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java @@ -148,7 +148,7 @@ public void setup() throws Exception { scmBlockLocationProtocol = Mockito.mock(ScmBlockLocationProtocol.class); metrics = Mockito.mock(OMPerformanceMetrics.class); keyManager = new KeyManagerImpl(ozoneManager, scmClient, ozoneConfiguration, - "", metrics); + metrics); when(ozoneManager.getScmClient()).thenReturn(scmClient); when(ozoneManager.getBlockTokenSecretManager()) .thenReturn(ozoneBlockTokenSecretManager);