diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java index c4e543554a6a..f28897341d4e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java @@ -19,6 +19,8 @@ package org.apache.hadoop.fs.ozone; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -29,6 +31,8 @@ import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -40,9 +44,13 @@ import java.net.URI; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; import static org.junit.Assert.fail; /** @@ -209,6 +217,118 @@ public void testObjectStoreCreateWithO3fs() throws Exception { } + + @Test + public void testKeyCreationFailDuetoDirectoryCreationBeforeCommit() + throws Exception { + OzoneVolume ozoneVolume = + cluster.getRpcClient().getObjectStore().getVolume(volumeName); + + OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); + + OzoneOutputStream ozoneOutputStream = + ozoneBucket.createKey("/a/b/c", 10); + byte[] b = new byte[10]; + Arrays.fill(b, (byte)96); + ozoneOutputStream.write(b); + + // Before close create directory with same name. + o3fs.mkdirs(new Path("/a/b/c")); + + try { + ozoneOutputStream.close(); + fail("testKeyCreationFailDuetoDirectoryCreationBeforeCommit"); + } catch (IOException ex) { + Assert.assertTrue(ex instanceof OMException); + Assert.assertEquals(NOT_A_FILE, + ((OMException) ex).getResult()); + } + + } + + + @Test + public void testMPUFailDuetoDirectoryCreationBeforeComplete() + throws Exception { + OzoneVolume ozoneVolume = + cluster.getRpcClient().getObjectStore().getVolume(volumeName); + + OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); + + String keyName = "/dir1/dir2/mpukey"; + OmMultipartInfo omMultipartInfo = + ozoneBucket.initiateMultipartUpload(keyName); + Assert.assertNotNull(omMultipartInfo); + + OzoneOutputStream ozoneOutputStream = + ozoneBucket.createMultipartKey(keyName, 10, 1, + omMultipartInfo.getUploadID()); + byte[] b = new byte[10]; + Arrays.fill(b, (byte)96); + ozoneOutputStream.write(b); + + // Before close, create directory with same name. + o3fs.mkdirs(new Path(keyName)); + + // This should succeed, as we check during creation of part or during + // complete MPU. + ozoneOutputStream.close(); + + Map partsMap = new HashMap<>(); + partsMap.put(1, ozoneOutputStream.getCommitUploadPartInfo().getPartName()); + + // Should fail, as we have directory with same name. + try { + ozoneBucket.completeMultipartUpload(keyName, + omMultipartInfo.getUploadID(), partsMap); + fail("testMPUFailDuetoDirectoryCreationBeforeComplete failed"); + } catch (OMException ex) { + Assert.assertTrue(ex instanceof OMException); + Assert.assertEquals(NOT_A_FILE, ex.getResult()); + } + + // Delete directory + o3fs.delete(new Path(keyName), true); + + // And try again for complete MPU. This should succeed. + ozoneBucket.completeMultipartUpload(keyName, + omMultipartInfo.getUploadID(), partsMap); + + try (FSDataInputStream ozoneInputStream = o3fs.open(new Path(keyName))) { + byte[] buffer = new byte[10]; + // This read will not change the offset inside the file + int readBytes = ozoneInputStream.read(0, buffer, 0, 10); + String readData = new String(buffer, 0, readBytes, UTF_8); + Assert.assertEquals(new String(b, 0, b.length, UTF_8), readData); + } + + } + + @Test + public void testCreateDirectoryFirstThenKeyAndFileWithSameName() + throws Exception { + o3fs.mkdirs(new Path("/t1/t2")); + + try { + o3fs.create(new Path("/t1/t2")); + fail("testCreateDirectoryFirstThenFileWithSameName failed"); + } catch (FileAlreadyExistsException ex) { + Assert.assertTrue(ex.getMessage().contains(NOT_A_FILE.name())); + } + + OzoneVolume ozoneVolume = + cluster.getRpcClient().getObjectStore().getVolume(volumeName); + OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); + ozoneBucket.createDirectory("t1/t2"); + try { + ozoneBucket.createKey("t1/t2", 0); + fail("testCreateDirectoryFirstThenFileWithSameName failed"); + } catch (OMException ex) { + Assert.assertTrue(ex instanceof OMException); + Assert.assertEquals(NOT_A_FILE, ex.getResult()); + } + } + private void checkPath(Path path) { try { o3fs.getFileStatus(path); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index dccb93bb9da3..8ee3f17618d7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -60,6 +60,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; /** @@ -158,6 +159,16 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, validateBucketAndVolume(omMetadataManager, volumeName, bucketName); + // Check for directory exists with same name, if it exists throw error. + if (ozoneManager.getEnableFileSystemPaths()) { + if (checkDirectoryAlreadyExists(volumeName, bucketName, keyName, + omMetadataManager)) { + throw new OMException("Can not create file: " + keyName + + " as there is already directory in the given path", NOT_A_FILE); + } + } + + omKeyInfo = omMetadataManager.getOpenKeyTable().get(dbOpenKey); if (omKeyInfo == null) { throw new OMException("Failed to commit key, as " + dbOpenKey + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index d863073cd524..f55cc5dcafaf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@ -530,4 +530,23 @@ protected FileEncryptionInfo getFileEncryptionInfo(KeyArgs keyArgs) { } return encryptionInfo; } + + /** + * Check directory exists. If exists return true, else false. + * @param volumeName + * @param bucketName + * @param keyName + * @param omMetadataManager + * @throws IOException + */ + protected boolean checkDirectoryAlreadyExists(String volumeName, + String bucketName, String keyName, OMMetadataManager omMetadataManager) + throws IOException { + if (omMetadataManager.getKeyTable().isExist( + omMetadataManager.getOzoneDirKey(volumeName, bucketName, + keyName))) { + return true; + } + return false; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 83cc28b01070..dff022ba80bf 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -55,6 +55,7 @@ import com.google.common.base.Optional; import org.apache.commons.codec.digest.DigestUtils; import static org.apache.hadoop.ozone.OzoneConsts.OM_MULTIPART_MIN_SIZE; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_A_FILE; import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -140,6 +141,15 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, OmMultipartKeyInfo multipartKeyInfo = omMetadataManager .getMultipartInfoTable().get(multipartKey); + // Check for directory exists with same name, if it exists throw error. + if (ozoneManager.getEnableFileSystemPaths()) { + if (checkDirectoryAlreadyExists(volumeName, bucketName, keyName, + omMetadataManager)) { + throw new OMException("Can not Complete MPU for file: " + keyName + + " as there is already directory in the given path", NOT_A_FILE); + } + } + if (multipartKeyInfo == null) { throw new OMException( failureMessage(requestedVolume, requestedBucket, keyName), diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index 1eac008365c7..a31986e7cae1 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -83,6 +83,7 @@ import org.apache.commons.lang3.tuple.Pair; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_DEFAULT; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_KEY; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.ENTITY_TOO_SMALL; @@ -198,6 +199,18 @@ public Response put( .build(); } catch (IOException ex) { LOG.error("Exception occurred in PutObject", ex); + if (ex instanceof OMException) { + if (((OMException) ex).getResult() == ResultCodes.NOT_A_FILE) { + OS3Exception os3Exception = S3ErrorTable.newError(INVALID_REQUEST, + keyPath); + os3Exception.setErrorMessage("An error occurred (InvalidRequest) " + + "when calling the PutObject/MPU PartUpload operation: " + + OZONE_OM_ENABLE_FILESYSTEM_PATHS + " is enabled Keys are" + + " considered as Unix Paths. Path has Violated FS Semantics " + + "which caused put operation to fail."); + throw os3Exception; + } + } throw ex; } finally { if (output != null) { @@ -522,6 +535,14 @@ public Response completeMultipartUpload(@PathParam("bucket") String bucket, "when calling the CompleteMultipartUpload operation: You must " + "specify at least one part"); throw os3Exception; + } else if(ex.getResult() == ResultCodes.NOT_A_FILE) { + OS3Exception os3Exception = S3ErrorTable.newError(INVALID_REQUEST, key); + os3Exception.setErrorMessage("An error occurred (InvalidRequest) " + + "when calling the CompleteMultipartUpload operation: " + + OZONE_OM_ENABLE_FILESYSTEM_PATHS + " is enabled Keys are " + + "considered as Unix Paths. A directory already exists with a " + + "given KeyName caused failure for MPU"); + throw os3Exception; } throw ex; }