Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ public final class OzoneConsts {
*/

public static final String OM_KEY_PREFIX = "/";
public static final String DOUBLE_SLASH_OM_KEY_PREFIX = "//";
public static final String OM_USER_PREFIX = "$";
public static final String OM_S3_PREFIX = "S3:";
public static final String OM_S3_CALLER_CONTEXT_PREFIX = "S3Auth:S3G|";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import static org.apache.hadoop.hdds.HddsUtils.getHostName;
import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
import static org.apache.hadoop.ozone.OzoneConsts.DOUBLE_SLASH_OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
Expand Down Expand Up @@ -760,7 +761,7 @@ public static String normalizeKey(String keyName,
if (!StringUtils.isBlank(keyName)) {
String normalizedKeyName;
if (keyName.startsWith(OM_KEY_PREFIX)) {
normalizedKeyName = new Path(keyName).toUri().getPath();
normalizedKeyName = new Path(normalizeLeadingSlashes(keyName)).toUri().getPath();
} else {
normalizedKeyName = new Path(OM_KEY_PREFIX + keyName)
.toUri().getPath();
Expand All @@ -778,6 +779,20 @@ public static String normalizeKey(String keyName,
return keyName;
}

/**
* Normalizes paths by replacing multiple leading slashes with a single slash.
*/
private static String normalizeLeadingSlashes(String keyName) {
if (keyName.startsWith(DOUBLE_SLASH_OM_KEY_PREFIX)) {
int index = 0;
while (index < keyName.length() && keyName.charAt(index) == OM_KEY_PREFIX.charAt(0)) {
index++;
}
return OM_KEY_PREFIX + keyName.substring(index);
}
return keyName;
}

/**
* Normalizes a given path up to the bucket level.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.s3.S3ClientFactory;
import org.apache.hadoop.ozone.s3.S3GatewayService;
import org.apache.ozone.test.OzoneTestBase;
Expand Down Expand Up @@ -365,6 +368,32 @@ public void testPutObject() {
assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag());
}

@Test
public void testPutDoubleSlashPrefixObject() throws IOException {
final String bucketName = getBucketName();
final String keyName = "//dir1";
final String content = "bar";
OzoneConfiguration conf = cluster.getConf();
// Create a FSO bucket for test
try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) {
ObjectStore store = ozoneClient.getObjectStore();
OzoneVolume volume = store.getS3Volume();
OmBucketInfo.Builder bucketInfo = new OmBucketInfo.Builder()
.setVolumeName(volume.getName())
.setBucketName(bucketName)
.setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED);
OzoneManagerProtocol ozoneManagerProtocol = store.getClientProxy().getOzoneManagerClient();
ozoneManagerProtocol.createBucket(bucketInfo.build());
}

InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata());
assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag());

S3Object object = s3Client.getObject(bucketName, keyName);
assertEquals(content.length(), object.getObjectMetadata().getContentLength());
}

@Test
public void testPutObjectEmpty() {
final String bucketName = getBucketName();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

import java.io.FileNotFoundException;
import java.io.IOException;
Expand Down Expand Up @@ -67,6 +68,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;

/**
* Class tests create with object store and getFileStatus.
Expand Down Expand Up @@ -386,6 +389,38 @@ public void testListKeysWithNotNormalizedPath() throws Exception {
checkKeyList(ozoneKeyIterator, keys);
}

@ParameterizedTest
@ValueSource(ints = {2, 3, 4})
public void testDoubleSlashPrefixPathNormalization(int slashCount) throws Exception {
OzoneVolume ozoneVolume = client.getObjectStore().getVolume(volumeName);
OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName);
// Generate a path with the specified number of leading slashes
StringBuilder keyPrefix = new StringBuilder();
for (int i = 0; i < slashCount; i++) {
keyPrefix.append('/');
}
String dirPath = "dir" + slashCount + "/";
String keyName = "key" + slashCount;
String slashyKey = keyPrefix + dirPath + keyName;
String normalizedKey = dirPath + keyName;
byte[] data = new byte[10];
Arrays.fill(data, (byte)96);
ArrayList<String> expectedKeys = new ArrayList<>();
expectedKeys.add(dirPath);
expectedKeys.add(normalizedKey);
TestDataUtil.createKey(ozoneBucket, slashyKey, data);

try {
ozoneBucket.readKey(slashyKey).close();
ozoneBucket.readKey(normalizedKey).close();
} catch (Exception e) {
fail("Should be able to read key " + e.getMessage());
}

Iterator<? extends OzoneKey> it = ozoneBucket.listKeys(dirPath);
checkKeyList(it, expectedKeys);
}

private void checkKeyList(Iterator<? extends OzoneKey > ozoneKeyIterator,
List<String> keys) {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ public void testNormalizePathsEnabled() throws Exception {
validateAndNormalizeKey(true, "a/b/c/d/"));
assertEquals("a/b/c/...../d",
validateAndNormalizeKey(true, "////a/b/////c/...../d/"));
assertEquals("a/b/c", validateAndNormalizeKey(true, "/a/b/c"));
assertEquals("a/b/c", validateAndNormalizeKey(true, "//a/b/c"));
assertEquals("a/b/c", validateAndNormalizeKey(true, "///a/b/c"));
}

@Test
Expand Down