diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index 016011078c8..3d21c31be74 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.fs.Trash; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; @@ -279,6 +280,7 @@ public void testFileSystem() throws Exception { testRenameDestinationParentDoesntExist(); testRenameToParentDir(); testSeekOnFileLength(); + testAllocateMoreThanOneBlock(); testDeleteRoot(); testRecursiveDelete(); @@ -585,6 +587,7 @@ protected void testListStatusOnRoot() throws Exception { */ protected void testListStatusOnLargeDirectory() throws Exception { Path root = new Path("/"); + deleteRootDir(); // cleanup Set paths = new TreeSet<>(); int numDirs = 5111; for(int i = 0; i < numDirs; i++) { @@ -595,6 +598,21 @@ protected void testListStatusOnLargeDirectory() throws Exception { } FileStatus[] fileStatuses = o3fs.listStatus(root); + // Added logs for debugging failures, to check any sub-path mismatches. + Set actualPaths = new TreeSet<>(); + ArrayList actualPathList = new ArrayList<>(); + if (rootItemCount != fileStatuses.length) { + for (int i = 0; i < fileStatuses.length; i++) { + actualPaths.add(fileStatuses[i].getPath().getName()); + actualPathList.add(fileStatuses[i].getPath().getName()); + } + if (rootItemCount != actualPathList.size()) { + actualPaths.removeAll(paths); + actualPathList.removeAll(paths); + LOG.info("actualPaths: {}", actualPaths); + LOG.info("actualPathList: {}", actualPathList); + } + } assertEquals( "Total directories listed do not match the existing directories", rootItemCount, fileStatuses.length); @@ -604,6 +622,31 @@ protected void testListStatusOnLargeDirectory() throws Exception { } } + /** + * Cleanup files and directories. + * + * @throws IOException DB failure + */ + protected void deleteRootDir() throws IOException { + Path root = new Path("/"); + FileStatus[] fileStatuses = fs.listStatus(root); + + rootItemCount = 0; // reset to zero + + if (fileStatuses == null) { + return; + } + + for (FileStatus fStatus : fileStatuses) { + fs.delete(fStatus.getPath(), true); + } + + fileStatuses = fs.listStatus(root); + if (fileStatuses != null) { + Assert.assertEquals("Delete root failed!", 0, fileStatuses.length); + } + } + /** * Tests listStatus on a path with subdirs. */ @@ -662,6 +705,38 @@ public void testSeekOnFileLength() throws IOException { } } + public void testAllocateMoreThanOneBlock() throws IOException { + Path file = new Path("/file"); + String str = "TestOzoneFileSystemV1.testSeekOnFileLength"; + byte[] strBytes = str.getBytes(); + long numBlockAllocationsOrg = + cluster.getOzoneManager().getMetrics().getNumBlockAllocates(); + + try (FSDataOutputStream out1 = fs.create(file, FsPermission.getDefault(), + true, 8, (short) 3, 1, null)) { + for (int i = 0; i < 100000; i++) { + out1.write(strBytes); + } + } + + try (FSDataInputStream stream = fs.open(file)) { + FileStatus fileStatus = fs.getFileStatus(file); + long blkSize = fileStatus.getBlockSize(); + long fileLength = fileStatus.getLen(); + Assert.assertTrue("Block allocation should happen", + fileLength > blkSize); + + long newNumBlockAllocations = + cluster.getOzoneManager().getMetrics().getNumBlockAllocates(); + + Assert.assertTrue("Block allocation should happen", + (newNumBlockAllocations > numBlockAllocationsOrg)); + + stream.seek(fileLength); + assertEquals(-1, stream.read()); + } + } + public void testDeleteRoot() throws IOException { Path dir = new Path("/dir"); fs.mkdirs(dir); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java index d66ec4dbe9b..dfbd7200059 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java @@ -38,7 +38,6 @@ import org.slf4j.LoggerFactory; import java.io.FileNotFoundException; -import java.io.IOException; import java.util.ArrayList; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; @@ -396,6 +395,9 @@ public void testFileSystem() throws Exception { testSeekOnFileLength(); deleteRootDir(); + testAllocateMoreThanOneBlock(); + deleteRootDir(); + testFileDelete(); deleteRootDir(); @@ -406,32 +408,6 @@ public void testFileSystem() throws Exception { deleteRootDir(); } - /** - * Cleanup files and directories. - * - * @throws IOException DB failure - */ - protected void deleteRootDir() throws IOException { - Path root = new Path("/"); - FileStatus[] fileStatuses = fs.listStatus(root); - - if (fileStatuses == null) { - return; - } - - for (FileStatus fStatus : fileStatuses) { - fs.delete(fStatus.getPath(), true); - } - - fileStatuses = fs.listStatus(root); - if (fileStatuses != null) { - Assert.assertEquals("Delete root failed!", 0, fileStatuses.length); - rootItemCount = 0; - return; - } - rootItemCount = 0; - } - @NotNull @Override protected OzoneConfiguration getOzoneConfig() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index 7b5988c388e..31120f13c5f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.om.request.file.OMFileCreateRequestV1; import org.apache.hadoop.ozone.om.request.key.OMKeysDeleteRequest; import org.apache.hadoop.ozone.om.request.key.OMAllocateBlockRequest; +import org.apache.hadoop.ozone.om.request.key.OMAllocateBlockRequestV1; import org.apache.hadoop.ozone.om.request.key.OMKeyCommitRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyCommitRequestV1; import org.apache.hadoop.ozone.om.request.key.OMKeyCreateRequest; @@ -138,6 +139,9 @@ public static OMClientRequest createClientRequest(OMRequest omRequest) { case SetBucketProperty: return new OMBucketSetPropertyRequest(omRequest); case AllocateBlock: + if (omLayoutVersionV1) { + return new OMAllocateBlockRequestV1(omRequest); + } return new OMAllocateBlockRequest(omRequest); case CreateKey: return new OMKeyCreateRequest(omRequest); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java index a14fae5e877..bff1d51ee4c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequestV1.java @@ -141,6 +141,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, pathInfoV1.getLeafNodeName()); dbFileInfo = OMFileRequest.getOmKeyInfoFromFileTable(false, omMetadataManager, dbFileKey, keyName); + if (dbFileInfo != null) { + ozoneManager.getKeyManager().refresh(dbFileInfo); + } } // check if the file or directory already existed in OM diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java index fc9bab0f2e8..aadc1266562 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java @@ -52,6 +52,8 @@ import javax.annotation.Nonnull; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; /** * Base class for file requests. @@ -851,4 +853,54 @@ private static boolean checkSubFileExists(OmKeyInfo omKeyInfo, public static boolean isImmediateChild(long parentId, long ancestorId) { return parentId == ancestorId; } + + /** + * Get parent id for the user given path. + * + * @param bucketId bucket id + * @param pathComponents fie path elements + * @param keyName user given key name + * @param omMetadataManager om metadata manager + * @return lastKnownParentID + * @throws IOException DB failure or parent not exists in DirectoryTable + */ + public static long getParentID(long bucketId, Iterator pathComponents, + String keyName, OMMetadataManager omMetadataManager) throws IOException { + + long lastKnownParentId = bucketId; + + // If no sub-dirs then bucketID is the root/parent. + if(!pathComponents.hasNext()){ + return bucketId; + } + + OmDirectoryInfo omDirectoryInfo; + while (pathComponents.hasNext()) { + String nodeName = pathComponents.next().toString(); + boolean reachedLastPathComponent = !pathComponents.hasNext(); + String dbNodeName = + omMetadataManager.getOzonePathKey(lastKnownParentId, nodeName); + + omDirectoryInfo = omMetadataManager. + getDirectoryTable().get(dbNodeName); + if (omDirectoryInfo != null) { + if (reachedLastPathComponent) { + throw new OMException("Can not create file: " + keyName + + " as there is already directory in the given path", + NOT_A_FILE); + } + lastKnownParentId = omDirectoryInfo.getObjectID(); + } else { + // One of the sub-dir doesn't exists in DB. Immediate parent should + // exists for committing the key, otherwise will fail the operation. + if (!reachedLastPathComponent) { + throw new OMException("Failed to find parent directory of " + + keyName + " in DirectoryTable", KEY_NOT_FOUND); + } + break; + } + } + + return lastKnownParentId; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java new file mode 100644 index 00000000000..55d3108ee09 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequestV1.java @@ -0,0 +1,230 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.key; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.audit.OMAction; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.key.OMAllocateBlockResponse; +import org.apache.hadoop.ozone.om.response.key.OMAllocateBlockResponseV1; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.AllocateBlockResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; +import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; + +/** + * Handles allocate block request layout version V1. + */ +public class OMAllocateBlockRequestV1 extends OMAllocateBlockRequest { + + private static final Logger LOG = + LoggerFactory.getLogger(OMAllocateBlockRequestV1.class); + + public OMAllocateBlockRequestV1(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, + long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) { + + AllocateBlockRequest allocateBlockRequest = + getOmRequest().getAllocateBlockRequest(); + + KeyArgs keyArgs = + allocateBlockRequest.getKeyArgs(); + + OzoneManagerProtocolProtos.KeyLocation blockLocation = + allocateBlockRequest.getKeyLocation(); + Preconditions.checkNotNull(blockLocation); + + String volumeName = keyArgs.getVolumeName(); + String bucketName = keyArgs.getBucketName(); + String keyName = keyArgs.getKeyName(); + long clientID = allocateBlockRequest.getClientID(); + + OMMetrics omMetrics = ozoneManager.getMetrics(); + omMetrics.incNumBlockAllocateCalls(); + + AuditLogger auditLogger = ozoneManager.getAuditLogger(); + + Map auditMap = buildKeyArgsAuditMap(keyArgs); + auditMap.put(OzoneConsts.CLIENT_ID, String.valueOf(clientID)); + + OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + String openKeyName = null; + + OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( + getOmRequest()); + OMClientResponse omClientResponse = null; + + OmKeyInfo openKeyInfo = null; + IOException exception = null; + OmVolumeArgs omVolumeArgs = null; + OmBucketInfo omBucketInfo = null; + boolean acquiredLock = false; + + try { + keyArgs = resolveBucketLink(ozoneManager, keyArgs, auditMap); + volumeName = keyArgs.getVolumeName(); + bucketName = keyArgs.getBucketName(); + + // check Acl + checkKeyAclsInOpenKeyTable(ozoneManager, volumeName, bucketName, keyName, + IAccessAuthorizer.ACLType.WRITE, allocateBlockRequest.getClientID()); + + validateBucketAndVolume(omMetadataManager, volumeName, + bucketName); + + // Here we don't acquire bucket/volume lock because for a single client + // allocateBlock is called in serial fashion. With this approach, it + // won't make 'fail-fast' during race condition case on delete/rename op, + // assuming that later it will fail at the key commit operation. + openKeyName = getOpenKeyName(volumeName, bucketName, keyName, clientID, + ozoneManager); + openKeyInfo = getOpenKeyInfo(omMetadataManager, openKeyName, keyName); + if (openKeyInfo == null) { + throw new OMException("Open Key not found " + openKeyName, + KEY_NOT_FOUND); + } + + List newLocationList = Collections.singletonList( + OmKeyLocationInfo.getFromProtobuf(blockLocation)); + omVolumeArgs = getVolumeInfo(omMetadataManager, volumeName); + + acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, + volumeName, bucketName); + omBucketInfo = getBucketInfo(omMetadataManager, volumeName, bucketName); + // check bucket and volume quota + long preAllocatedSpace = newLocationList.size() + * ozoneManager.getScmBlockSize() + * openKeyInfo.getFactor().getNumber(); + checkBucketQuotaInBytes(omBucketInfo, preAllocatedSpace); + // Append new block + openKeyInfo.appendNewBlocks(newLocationList, false); + + // Set modification time. + openKeyInfo.setModificationTime(keyArgs.getModificationTime()); + + // Set the UpdateID to current transactionLogIndex + openKeyInfo.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); + + // Add to cache. + addOpenTableCacheEntry(trxnLogIndex, omMetadataManager, openKeyName, + openKeyInfo); + omBucketInfo.incrUsedBytes(preAllocatedSpace); + + omResponse.setAllocateBlockResponse(AllocateBlockResponse.newBuilder() + .setKeyLocation(blockLocation).build()); + omClientResponse = getOmClientResponse(clientID, omResponse, + openKeyInfo, omVolumeArgs, omBucketInfo.copyObject()); + LOG.debug("Allocated block for Volume:{}, Bucket:{}, OpenKey:{}", + volumeName, bucketName, openKeyName); + } catch (IOException ex) { + omMetrics.incNumBlockAllocateCallFails(); + exception = ex; + omClientResponse = new OMAllocateBlockResponse(createErrorOMResponse( + omResponse, exception)); + LOG.error("Allocate Block failed. Volume:{}, Bucket:{}, OpenKey:{}. " + + "Exception:{}", volumeName, bucketName, openKeyName, exception); + } finally { + addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, + omDoubleBufferHelper); + if (acquiredLock) { + omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName, + bucketName); + } + } + + auditLog(auditLogger, buildAuditMessage(OMAction.ALLOCATE_BLOCK, auditMap, + exception, getOmRequest().getUserInfo())); + + return omClientResponse; + } + + private OmKeyInfo getOpenKeyInfo(OMMetadataManager omMetadataManager, + String openKeyName, String keyName) throws IOException { + String fileName = OzoneFSUtils.getFileName(keyName); + return OMFileRequest.getOmKeyInfoFromFileTable(true, + omMetadataManager, openKeyName, fileName); + } + + private String getOpenKeyName(String volumeName, String bucketName, + String keyName, long clientID, OzoneManager ozoneManager) + throws IOException { + OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + OmBucketInfo omBucketInfo = + omMetadataManager.getBucketTable().get(bucketKey); + long bucketId = omBucketInfo.getObjectID(); + String fileName = OzoneFSUtils.getFileName(keyName); + Iterator pathComponents = Paths.get(keyName).iterator(); + long parentID = OMFileRequest.getParentID(bucketId, pathComponents, + keyName, omMetadataManager); + return omMetadataManager.getOpenFileName(parentID, fileName, + clientID); + } + + private void addOpenTableCacheEntry(long trxnLogIndex, + OMMetadataManager omMetadataManager, String openKeyName, + OmKeyInfo openKeyInfo) { + String fileName = openKeyInfo.getFileName(); + OMFileRequest.addOpenFileTableCacheEntry(omMetadataManager, openKeyName, + openKeyInfo, fileName, trxnLogIndex); + } + + @NotNull + private OMClientResponse getOmClientResponse(long clientID, + OMResponse.Builder omResponse, OmKeyInfo openKeyInfo, + OmVolumeArgs omVolumeArgs, OmBucketInfo omBucketInfo) { + return new OMAllocateBlockResponseV1(omResponse.build(), + openKeyInfo, clientID, omVolumeArgs, omBucketInfo); + } +} \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java index ffabcd74726..34417cae00e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestV1.java @@ -25,7 +25,6 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; -import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -53,7 +52,6 @@ import java.util.Map; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; -import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; /** @@ -130,8 +128,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, String fileName = OzoneFSUtils.getFileName(keyName); omBucketInfo = omMetadataManager.getBucketTable().get(bucketKey); long bucketId = omBucketInfo.getObjectID(); - long parentID = getParentID(bucketId, pathComponents, keyName, - omMetadataManager, ozoneManager); + long parentID = OMFileRequest.getParentID(bucketId, pathComponents, + keyName, omMetadataManager); String dbFileKey = omMetadataManager.getOzonePathKey(parentID, fileName); dbOpenFileKey = omMetadataManager.getOpenFileName(parentID, fileName, commitKeyRequest.getClientID()); @@ -199,74 +197,4 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, return omClientResponse; } - - - /** - * Check for directory exists with same name, if it exists throw error. - * - * @param keyName key name - * @param ozoneManager Ozone Manager - * @param reachedLastPathComponent true if the path component is a fileName - * @throws IOException if directory exists with same name - */ - private void checkDirectoryAlreadyExists(String keyName, - OzoneManager ozoneManager, - boolean reachedLastPathComponent) - throws IOException { - // Reached last component, which would be a file. Returns its parentID. - if (reachedLastPathComponent && ozoneManager.getEnableFileSystemPaths()) { - throw new OMException("Can not create file: " + keyName + - " as there is already directory in the given path", NOT_A_FILE); - } - } - - /** - * Get parent id for the user given path. - * - * @param bucketId bucket id - * @param pathComponents fie path elements - * @param keyName user given key name - * @param omMetadataManager metadata manager - * @return lastKnownParentID - * @throws IOException DB failure or parent not exists in DirectoryTable - */ - private long getParentID(long bucketId, Iterator pathComponents, - String keyName, OMMetadataManager omMetadataManager, - OzoneManager ozoneManager) - throws IOException { - - long lastKnownParentId = bucketId; - - // If no sub-dirs then bucketID is the root/parent. - if(!pathComponents.hasNext()){ - return bucketId; - } - - OmDirectoryInfo omDirectoryInfo; - while (pathComponents.hasNext()) { - String nodeName = pathComponents.next().toString(); - boolean reachedLastPathComponent = !pathComponents.hasNext(); - String dbNodeName = - omMetadataManager.getOzonePathKey(lastKnownParentId, nodeName); - - omDirectoryInfo = omMetadataManager. - getDirectoryTable().get(dbNodeName); - if (omDirectoryInfo != null) { - checkDirectoryAlreadyExists(keyName, ozoneManager, - reachedLastPathComponent); - lastKnownParentId = omDirectoryInfo.getObjectID(); - } else { - // One of the sub-dir doesn't exists in DB. Immediate parent should - // exists for committing the key, otherwise will fail the operation. - if (!reachedLastPathComponent) { - throw new OMException("Failed to commit key, as parent directory of " - + keyName + " entry is not found in DirectoryTable", - KEY_NOT_FOUND); - } - break; - } - } - - return lastKnownParentId; - } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponse.java index acc43eef898..607a77925de 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponse.java @@ -77,4 +77,20 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, omMetadataManager.getBucketKey(omVolumeArgs.getVolume(), omBucketInfo.getBucketName()), omBucketInfo); } + + protected OmKeyInfo getOmKeyInfo() { + return omKeyInfo; + } + + protected OmVolumeArgs getOmVolumeArgs() { + return omVolumeArgs; + } + + protected OmBucketInfo getOmBucketInfo() { + return omBucketInfo; + } + + protected long getClientID() { + return clientID; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java new file mode 100644 index 00000000000..cf92e2b43d4 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMAllocateBlockResponseV1.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.key; + +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; + +import javax.annotation.Nonnull; +import java.io.IOException; + +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.OPEN_FILE_TABLE; + +/** + * Response for AllocateBlock request layout version V1. + */ +@CleanupTableInfo(cleanupTables = {OPEN_FILE_TABLE}) +public class OMAllocateBlockResponseV1 extends OMAllocateBlockResponse { + + public OMAllocateBlockResponseV1(@Nonnull OMResponse omResponse, + @Nonnull OmKeyInfo omKeyInfo, long clientID, + @Nonnull OmVolumeArgs omVolumeArgs, @Nonnull OmBucketInfo omBucketInfo) { + super(omResponse, omKeyInfo, clientID, omVolumeArgs, omBucketInfo); + } + + @Override + public void addToDBBatch(OMMetadataManager omMetadataManager, + BatchOperation batchOperation) throws IOException { + + OMFileRequest.addToOpenFileTable(omMetadataManager, batchOperation, + getOmKeyInfo(), getClientID()); + + // update bucket usedBytes. + omMetadataManager.getBucketTable().putWithBatch(batchOperation, + omMetadataManager.getBucketKey(getOmVolumeArgs().getVolume(), + getOmBucketInfo().getBucketName()), getOmBucketInfo()); + } +} + diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequest.java index 5010d0a7e3f..43cbc85a15b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequest.java @@ -397,28 +397,6 @@ protected OMRequest createFileRequest( } - /** - * Verify path in open key table. Also, it returns OMKeyInfo for the given - * key path. - * - * @param key key name - * @param id client id - * @param doAssert if true then do assertion, otherwise it just skip. - * @return om key info for the given key path. - * @throws Exception DB failure - */ - protected OmKeyInfo verifyPathInOpenKeyTable(String key, long id, - boolean doAssert) - throws Exception { - String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, - key, id); - OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable().get(openKey); - if (doAssert) { - Assert.assertNotNull("Failed to find key in OpenKeyTable", omKeyInfo); - } - return omKeyInfo; - } - /** * Gets OMFileCreateRequest reference. * diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java index 7ded386fdd8..046ac90e9a6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMFileCreateRequestV1.java @@ -23,6 +23,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.util.StringUtils; @@ -183,6 +184,10 @@ private OmDirectoryInfo getDirInfo(String key) protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java index f309da041a7..ae992f82e72 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequest.java @@ -24,6 +24,7 @@ import java.util.UUID; import org.apache.hadoop.ozone.OzoneConsts; +import org.jetbrains.annotations.NotNull; import org.junit.Assert; import org.junit.Test; @@ -57,21 +58,19 @@ public void testValidateAndUpdateCache() throws Exception { TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager); - TestOMRequestUtils.addKeyToTable(true, volumeName, bucketName, keyName, - clientID, replicationType, replicationFactor, omMetadataManager); + addKeyToOpenKeyTable(volumeName, bucketName); OMRequest modifiedOmRequest = doPreExecute(createAllocateBlockRequest()); OMAllocateBlockRequest omAllocateBlockRequest = - new OMAllocateBlockRequest(modifiedOmRequest); + getOmAllocateBlockRequest(modifiedOmRequest); // Check before calling validateAndUpdateCache. As adding DB entry has // not added any blocks, so size should be zero. - OmKeyInfo omKeyInfo = - omMetadataManager.getOpenKeyTable().get(omMetadataManager.getOpenKey( - volumeName, bucketName, keyName, clientID)); + OmKeyInfo omKeyInfo = verifyPathInOpenKeyTable(keyName, clientID, + true); List omKeyLocationInfo = omKeyInfo.getLatestVersionLocations().getLocationList(); @@ -87,10 +86,8 @@ public void testValidateAndUpdateCache() throws Exception { // Check open table whether new block is added or not. - omKeyInfo = - omMetadataManager.getOpenKeyTable().get(omMetadataManager.getOpenKey( - volumeName, bucketName, keyName, clientID)); - + omKeyInfo = verifyPathInOpenKeyTable(keyName, clientID, + true); // Check modification time Assert.assertEquals(modifiedOmRequest.getAllocateBlockRequest() @@ -119,6 +116,12 @@ public void testValidateAndUpdateCache() throws Exception { } + @NotNull + protected OMAllocateBlockRequest getOmAllocateBlockRequest( + OMRequest modifiedOmRequest) { + return new OMAllocateBlockRequest(modifiedOmRequest); + } + @Test public void testValidateAndUpdateCacheWithVolumeNotFound() throws Exception { @@ -126,7 +129,7 @@ public void testValidateAndUpdateCacheWithVolumeNotFound() throws Exception { doPreExecute(createAllocateBlockRequest()); OMAllocateBlockRequest omAllocateBlockRequest = - new OMAllocateBlockRequest(modifiedOmRequest); + getOmAllocateBlockRequest(modifiedOmRequest); OMClientResponse omAllocateBlockResponse = @@ -145,7 +148,7 @@ public void testValidateAndUpdateCacheWithBucketNotFound() throws Exception { doPreExecute(createAllocateBlockRequest()); OMAllocateBlockRequest omAllocateBlockRequest = - new OMAllocateBlockRequest(modifiedOmRequest); + getOmAllocateBlockRequest(modifiedOmRequest); // Added only volume to DB. @@ -168,7 +171,7 @@ public void testValidateAndUpdateCacheWithKeyNotFound() throws Exception { doPreExecute(createAllocateBlockRequest()); OMAllocateBlockRequest omAllocateBlockRequest = - new OMAllocateBlockRequest(modifiedOmRequest); + getOmAllocateBlockRequest(modifiedOmRequest); // Add volume, bucket entries to DB. TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName, @@ -190,10 +193,11 @@ public void testValidateAndUpdateCacheWithKeyNotFound() throws Exception { * @return OMRequest - modified request returned from preExecute. * @throws Exception */ - private OMRequest doPreExecute(OMRequest originalOMRequest) throws Exception { + protected OMRequest doPreExecute(OMRequest originalOMRequest) + throws Exception { OMAllocateBlockRequest omAllocateBlockRequest = - new OMAllocateBlockRequest(originalOMRequest); + getOmAllocateBlockRequest(originalOMRequest); OMRequest modifiedOmRequest = omAllocateBlockRequest.preExecute(ozoneManager); @@ -228,7 +232,7 @@ private OMRequest doPreExecute(OMRequest originalOMRequest) throws Exception { } - private OMRequest createAllocateBlockRequest() { + protected OMRequest createAllocateBlockRequest() { KeyArgs keyArgs = KeyArgs.newBuilder() .setVolumeName(volumeName).setBucketName(bucketName) @@ -246,4 +250,12 @@ private OMRequest createAllocateBlockRequest() { .setAllocateBlockRequest(allocateBlockRequest).build(); } + + protected String addKeyToOpenKeyTable(String volumeName, String bucketName) + throws Exception { + TestOMRequestUtils.addKeyToTable(true, volumeName, bucketName, + keyName, clientID, replicationType, replicationFactor, + omMetadataManager); + return ""; + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java new file mode 100644 index 00000000000..4e74979de2e --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMAllocateBlockRequestV1.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.ozone.om.request.key; + + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Time; +import org.jetbrains.annotations.NotNull; +import org.junit.Assert; + +/** + * Tests OMAllocateBlockRequest class layout version V1. + */ +public class TestOMAllocateBlockRequestV1 extends TestOMAllocateBlockRequest { + + @NotNull + @Override + protected OzoneConfiguration getOzoneConfiguration() { + OzoneConfiguration config = super.getOzoneConfiguration(); + config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + return config; + } + + protected String addKeyToOpenKeyTable(String volumeName, String bucketName) + throws Exception { + // need to initialize parentID + String parentDir = keyName; + String fileName = "file1"; + keyName = parentDir + OzoneConsts.OM_KEY_PREFIX + fileName; + + // add parentDir to dirTable + long parentID = TestOMRequestUtils.addParentsToDirTable(volumeName, + bucketName, parentDir, omMetadataManager); + long txnId = 50; + long objectId = parentID + 1; + + OmKeyInfo omKeyInfoV1 = + TestOMRequestUtils.createOmKeyInfo(volumeName, bucketName, keyName, + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, objectId, parentID, txnId, + Time.now()); + + // add key to openFileTable + TestOMRequestUtils.addFileToKeyTable(true, false, + fileName, omKeyInfoV1, clientID, txnLogId, omMetadataManager); + + return omMetadataManager.getOzonePathKey(parentID, fileName); + } + + @NotNull + protected OMAllocateBlockRequest getOmAllocateBlockRequest( + OzoneManagerProtocolProtos.OMRequest modifiedOmRequest) { + return new OMAllocateBlockRequestV1(modifiedOmRequest); + } + + @Override + protected OmKeyInfo verifyPathInOpenKeyTable(String key, long id, + boolean doAssert) throws Exception { + long bucketId = TestOMRequestUtils.getBucketId(volumeName, bucketName, + omMetadataManager); + String[] pathComponents = StringUtils.split(key, '/'); + long parentId = bucketId; + for (int indx = 0; indx < pathComponents.length; indx++) { + String pathElement = pathComponents[indx]; + // Reached last component, which is file name + if (indx == pathComponents.length - 1) { + String dbOpenFileName = omMetadataManager.getOpenFileName( + parentId, pathElement, id); + OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable() + .get(dbOpenFileName); + if (doAssert) { + Assert.assertNotNull("Invalid key!", omKeyInfo); + } + return omKeyInfo; + } else { + // directory + String dbKey = omMetadataManager.getOzonePathKey(parentId, + pathElement); + OmDirectoryInfo dirInfo = + omMetadataManager.getDirectoryTable().get(dbKey); + parentId = dirInfo.getObjectID(); + } + } + if (doAssert) { + Assert.fail("Invalid key!"); + } + return null; + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java index f5168e1a3f0..ed1e2bd3b77 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequestV1.java @@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.util.Time; @@ -87,6 +88,10 @@ protected String addKeyToOpenKeyTable() throws Exception { protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java index dbba1434323..7527e78e71a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyDeleteRequestV1.java @@ -18,8 +18,11 @@ package org.apache.hadoop.ozone.om.request.key; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.util.Time; @@ -54,4 +57,15 @@ protected String addKeyToTable() throws Exception { fileName, omKeyInfo, -1, 50, omMetadataManager); return omKeyInfo.getPath(); } + + @Override + protected OzoneConfiguration getOzoneConfiguration() { + OzoneConfiguration config = super.getOzoneConfiguration(); + config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + return config; + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java index 5f15c504776..178a07205e6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java @@ -26,11 +26,13 @@ import org.apache.hadoop.ozone.om.ResolvedBucket; import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.KeyManagerImpl; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; import org.jetbrains.annotations.NotNull; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.rules.TemporaryFolder; @@ -176,6 +178,29 @@ protected OzoneConfiguration getOzoneConfiguration() { return new OzoneConfiguration(); } + + /** + * Verify path in open key table. Also, it returns OMKeyInfo for the given + * key path. + * + * @param key key name + * @param id client id + * @param doAssert if true then do assertion, otherwise it just skip. + * @return om key info for the given key path. + * @throws Exception DB failure + */ + protected OmKeyInfo verifyPathInOpenKeyTable(String key, long id, + boolean doAssert) + throws Exception { + String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, + key, id); + OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable().get(openKey); + if (doAssert) { + Assert.assertNotNull("Failed to find key in OpenKeyTable", omKeyInfo); + } + return omKeyInfo; + } + @After public void stop() { omMetrics.unRegister(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java index d2ab4658242..2f8bb743da4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java @@ -23,6 +23,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse; import org.apache.hadoop.ozone.om.response.key.TestOMKeyCreateResponse; @@ -69,6 +70,10 @@ protected OMKeyCreateResponse getOmKeyCreateResponse(OmKeyInfo keyInfo, protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponse.java index 494a308b28e..1ac483d56be 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponse.java @@ -21,6 +21,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.util.Time; +import org.jetbrains.annotations.NotNull; import org.junit.Assert; import org.junit.Test; @@ -38,8 +39,7 @@ public class TestOMAllocateBlockResponse extends TestOMKeyResponse { @Test public void testAddToDBBatch() throws Exception { - OmKeyInfo omKeyInfo = TestOMRequestUtils.createOmKeyInfo(volumeName, - bucketName, keyName, replicationType, replicationFactor); + OmKeyInfo omKeyInfo = createOmKeyInfo(); OmVolumeArgs omVolumeArgs = OmVolumeArgs.newBuilder() .setOwnerName(keyName).setAdminName(keyName) .setVolume(volumeName).setCreationTime(Time.now()).build(); @@ -54,11 +54,10 @@ public void testAddToDBBatch() throws Exception { .setCmdType(OzoneManagerProtocolProtos.Type.AllocateBlock) .build(); OMAllocateBlockResponse omAllocateBlockResponse = - new OMAllocateBlockResponse(omResponse, omKeyInfo, clientID, - omVolumeArgs, omBucketInfo); + getOmAllocateBlockResponse(omKeyInfo, omVolumeArgs, + omBucketInfo, omResponse); - String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, - keyName, clientID); + String openKey = getOpenKey(); // Not adding key entry before to test whether commit is successful or not. Assert.assertFalse(omMetadataManager.getOpenKeyTable().isExist(openKey)); @@ -72,8 +71,7 @@ public void testAddToDBBatch() throws Exception { @Test public void testAddToDBBatchWithErrorResponse() throws Exception { - OmKeyInfo omKeyInfo = TestOMRequestUtils.createOmKeyInfo(volumeName, - bucketName, keyName, replicationType, replicationFactor); + OmKeyInfo omKeyInfo = createOmKeyInfo(); OmVolumeArgs omVolumeArgs = OmVolumeArgs.newBuilder() .setOwnerName(keyName).setAdminName(keyName) .setVolume(volumeName).setCreationTime(Time.now()).build(); @@ -88,12 +86,11 @@ public void testAddToDBBatchWithErrorResponse() throws Exception { .setCmdType(OzoneManagerProtocolProtos.Type.AllocateBlock) .build(); OMAllocateBlockResponse omAllocateBlockResponse = - new OMAllocateBlockResponse(omResponse, omKeyInfo, clientID, - omVolumeArgs, omBucketInfo); + getOmAllocateBlockResponse(omKeyInfo, omVolumeArgs, omBucketInfo, + omResponse); // Before calling addToDBBatch - String openKey = omMetadataManager.getOpenKey(volumeName, bucketName, - keyName, clientID); + String openKey = getOpenKey(); Assert.assertFalse(omMetadataManager.getOpenKeyTable().isExist(openKey)); omAllocateBlockResponse.checkAndUpdateDB(omMetadataManager, batchOperation); @@ -105,4 +102,22 @@ public void testAddToDBBatchWithErrorResponse() throws Exception { Assert.assertFalse(omMetadataManager.getOpenKeyTable().isExist(openKey)); } + + protected OmKeyInfo createOmKeyInfo() throws Exception { + return TestOMRequestUtils.createOmKeyInfo(volumeName, + bucketName, keyName, replicationType, replicationFactor); + } + + protected String getOpenKey() throws Exception { + return omMetadataManager.getOpenKey(volumeName, bucketName, + keyName, clientID); + } + + @NotNull + protected OMAllocateBlockResponse getOmAllocateBlockResponse( + OmKeyInfo omKeyInfo, OmVolumeArgs omVolumeArgs, + OmBucketInfo omBucketInfo, OMResponse omResponse) { + return new OMAllocateBlockResponse(omResponse, omKeyInfo, clientID, + omVolumeArgs, omBucketInfo); + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java new file mode 100644 index 00000000000..773edaa0afa --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMAllocateBlockResponseV1.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.key; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; +import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.util.Time; +import org.jetbrains.annotations.NotNull; + +/** + * Tests OMAllocateBlockResponse layout version V1. + */ +public class TestOMAllocateBlockResponseV1 + extends TestOMAllocateBlockResponse { + + // logical ID, which really doesn't exist in dirTable + private long parentID = 10; + private String fileName = "file1"; + + protected OmKeyInfo createOmKeyInfo() throws Exception { + // need to initialize parentID + String parentDir = keyName; + keyName = parentDir + OzoneConsts.OM_KEY_PREFIX + fileName; + + long txnId = 50; + long objectId = parentID + 1; + + OmKeyInfo omKeyInfoV1 = + TestOMRequestUtils.createOmKeyInfo(volumeName, bucketName, keyName, + HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.ONE, objectId, parentID, txnId, + Time.now()); + return omKeyInfoV1; + } + + protected String getOpenKey() throws Exception { + return omMetadataManager.getOpenFileName( + parentID, fileName, clientID); + } + + @NotNull + protected OMAllocateBlockResponse getOmAllocateBlockResponse( + OmKeyInfo omKeyInfo, OmVolumeArgs omVolumeArgs, + OmBucketInfo omBucketInfo, OMResponse omResponse) { + return new OMAllocateBlockResponseV1(omResponse, omKeyInfo, clientID, + omVolumeArgs, omBucketInfo); + } + + @NotNull + @Override + protected OzoneConfiguration getOzoneConfiguration() { + OzoneConfiguration config = super.getOzoneConfiguration(); + config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + return config; + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java index 369faa96b0c..1e59ce8eaa7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponseV1.java @@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.util.Time; @@ -96,6 +97,10 @@ protected String getOzoneKey() { protected OzoneConfiguration getOzoneConfiguration() { OzoneConfiguration config = super.getOzoneConfiguration(); config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); return config; } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java index 37be730a24d..967fa102b91 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyDeleteResponseV1.java @@ -18,12 +18,16 @@ package org.apache.hadoop.ozone.om.response.key; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.util.Time; +import org.jetbrains.annotations.NotNull; import org.junit.Assert; /** @@ -69,4 +73,16 @@ protected OmKeyInfo getOmKeyInfo() { getOmBucketInfo().getObjectID() + 1, getOmBucketInfo().getObjectID(), 100, Time.now()); } + + @NotNull + @Override + protected OzoneConfiguration getOzoneConfiguration() { + OzoneConfiguration config = super.getOzoneConfiguration(); + config.set(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, "V1"); + // omLayoutVersionV1 flag will be set while invoking OzoneManager#start() + // and its not invoked in this test. Hence it is explicitly setting + // this configuration to populate prefix tables. + OzoneManagerRatisUtils.setOmLayoutVersionV1(true); + return config; + } }