diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index d03cc2a22fe5..11065a27ec34 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -170,6 +170,7 @@ public final class OzoneConsts { */ public static final String OM_KEY_PREFIX = "/"; + public static final String DOUBLE_SLASH_OM_KEY_PREFIX = "//"; public static final String OM_USER_PREFIX = "$"; public static final String OM_S3_PREFIX = "S3:"; public static final String OM_S3_CALLER_CONTEXT_PREFIX = "S3Auth:S3G|"; diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 100ff74ed5e6..e71e5b912ac6 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.hdds.HddsUtils.getHostName; import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys; import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys; +import static org.apache.hadoop.ozone.OzoneConsts.DOUBLE_SLASH_OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; @@ -760,7 +761,7 @@ public static String normalizeKey(String keyName, if (!StringUtils.isBlank(keyName)) { String normalizedKeyName; if (keyName.startsWith(OM_KEY_PREFIX)) { - normalizedKeyName = new Path(keyName).toUri().getPath(); + normalizedKeyName = new Path(normalizeLeadingSlashes(keyName)).toUri().getPath(); } else { normalizedKeyName = new Path(OM_KEY_PREFIX + keyName) .toUri().getPath(); @@ -778,6 +779,20 @@ public static String normalizeKey(String keyName, return keyName; } + /** + * Normalizes paths by replacing multiple leading slashes with a single slash. + */ + private static String normalizeLeadingSlashes(String keyName) { + if (keyName.startsWith(DOUBLE_SLASH_OM_KEY_PREFIX)) { + int index = 0; + while (index < keyName.length() && keyName.charAt(index) == OM_KEY_PREFIX.charAt(0)) { + index++; + } + return OM_KEY_PREFIX + keyName.substring(index); + } + return keyName; + } + /** * Normalizes a given path up to the bucket level. * diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index a4eefb81dba7..c45b37eed911 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -101,6 +101,9 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.s3.S3ClientFactory; import org.apache.hadoop.ozone.s3.S3GatewayService; import org.apache.ozone.test.OzoneTestBase; @@ -365,6 +368,32 @@ public void testPutObject() { assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); } + @Test + public void testPutDoubleSlashPrefixObject() throws IOException { + final String bucketName = getBucketName(); + final String keyName = "//dir1"; + final String content = "bar"; + OzoneConfiguration conf = cluster.getConf(); + // Create a FSO bucket for test + try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) { + ObjectStore store = ozoneClient.getObjectStore(); + OzoneVolume volume = store.getS3Volume(); + OmBucketInfo.Builder bucketInfo = new OmBucketInfo.Builder() + .setVolumeName(volume.getName()) + .setBucketName(bucketName) + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED); + OzoneManagerProtocol ozoneManagerProtocol = store.getClientProxy().getOzoneManagerClient(); + ozoneManagerProtocol.createBucket(bucketInfo.build()); + } + + InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); + PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata()); + assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag()); + + S3Object object = s3Client.getObject(bucketName, keyName); + assertEquals(content.length(), object.getObjectMetadata().getContentLength()); + } + @Test public void testPutObjectEmpty() { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java index 0f5f24fd6609..26f06f9abc7d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java @@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.FileNotFoundException; import java.io.IOException; @@ -67,6 +68,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * Class tests create with object store and getFileStatus. @@ -386,6 +389,38 @@ public void testListKeysWithNotNormalizedPath() throws Exception { checkKeyList(ozoneKeyIterator, keys); } + @ParameterizedTest + @ValueSource(ints = {2, 3, 4}) + public void testDoubleSlashPrefixPathNormalization(int slashCount) throws Exception { + OzoneVolume ozoneVolume = client.getObjectStore().getVolume(volumeName); + OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); + // Generate a path with the specified number of leading slashes + StringBuilder keyPrefix = new StringBuilder(); + for (int i = 0; i < slashCount; i++) { + keyPrefix.append('/'); + } + String dirPath = "dir" + slashCount + "/"; + String keyName = "key" + slashCount; + String slashyKey = keyPrefix + dirPath + keyName; + String normalizedKey = dirPath + keyName; + byte[] data = new byte[10]; + Arrays.fill(data, (byte)96); + ArrayList expectedKeys = new ArrayList<>(); + expectedKeys.add(dirPath); + expectedKeys.add(normalizedKey); + TestDataUtil.createKey(ozoneBucket, slashyKey, data); + + try { + ozoneBucket.readKey(slashyKey).close(); + ozoneBucket.readKey(normalizedKey).close(); + } catch (Exception e) { + fail("Should be able to read key " + e.getMessage()); + } + + Iterator it = ozoneBucket.listKeys(dirPath); + checkKeyList(it, expectedKeys); + } + private void checkKeyList(Iterator ozoneKeyIterator, List keys) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestNormalizePaths.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestNormalizePaths.java index c37cbfda5945..0889a532727f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestNormalizePaths.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestNormalizePaths.java @@ -58,6 +58,9 @@ public void testNormalizePathsEnabled() throws Exception { validateAndNormalizeKey(true, "a/b/c/d/")); assertEquals("a/b/c/...../d", validateAndNormalizeKey(true, "////a/b/////c/...../d/")); + assertEquals("a/b/c", validateAndNormalizeKey(true, "/a/b/c")); + assertEquals("a/b/c", validateAndNormalizeKey(true, "//a/b/c")); + assertEquals("a/b/c", validateAndNormalizeKey(true, "///a/b/c")); } @Test