From 4501dcc9d692b122a69f0a2fbe886663a8a23dbc Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Thu, 30 Jul 2020 15:36:07 -0700 Subject: [PATCH 1/2] HDDS-4012. Normalize Keypath for lookupKey. --- .../TestOzoneFSWithObjectStoreCreate.java | 40 +++++++++++++++++++ .../hadoop/ozone/om/KeyManagerImpl.java | 11 ++++- .../apache/hadoop/ozone/om/OzoneManager.java | 1 - .../ozone/om/request/OMClientRequest.java | 2 +- 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java index c4e543554a6a..476c1fc8f9d0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.ozone; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -27,6 +28,7 @@ import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.junit.Assert; @@ -209,6 +211,44 @@ public void testObjectStoreCreateWithO3fs() throws Exception { } + @Test + public void testReadWithNotNormalizedPath() throws Exception { + OzoneVolume ozoneVolume = + cluster.getRpcClient().getObjectStore().getVolume(volumeName); + + OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); + + String key = "/dir1///dir2/file1/"; + + int length = 10; + byte[] input = new byte[length]; + Arrays.fill(input, (byte)96); + String inputString = new String(input); + + OzoneOutputStream ozoneOutputStream = + ozoneBucket.createKey(key, length); + + ozoneOutputStream.write(input); + ozoneOutputStream.write(input, 0, 10); + ozoneOutputStream.close(); + + // Read the key with given key name. + OzoneInputStream ozoneInputStream = ozoneBucket.readKey(key); + byte[] read = new byte[length]; + ozoneInputStream.read(read, 0, length); + ozoneInputStream.close(); + + Assert.assertEquals(inputString, new String(read)); + + // Read using filesystem. + FSDataInputStream fsDataInputStream = o3fs.open(new Path(key)); + read = new byte[length]; + fsDataInputStream.read(read, 0, length); + ozoneInputStream.close(); + + Assert.assertEquals(inputString, new String(read)); + } + private void checkPath(Path path) { try { o3fs.getFileStatus(path); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 66ddeb64dd07..40273581ff22 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -94,6 +94,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; import org.apache.hadoop.ozone.security.OzoneBlockTokenSecretManager; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; @@ -165,6 +166,8 @@ public class KeyManagerImpl implements KeyManager { private final KeyProviderCryptoExtension kmsProvider; private final PrefixManager prefixManager; + private final boolean enableFileSystemPaths; + @VisibleForTesting public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient, @@ -208,6 +211,9 @@ public KeyManagerImpl(OzoneManager om, ScmClient scmClient, this.listTrashKeysMax = conf.getInt( OZONE_CLIENT_LIST_TRASH_KEYS_MAX, OZONE_CLIENT_LIST_TRASH_KEYS_MAX_DEFAULT); + this.enableFileSystemPaths = + conf.getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, + OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT); this.ozoneManager = om; this.omId = omId; @@ -645,7 +651,10 @@ public OmKeyInfo lookupKey(OmKeyArgs args, String clientAddress) Preconditions.checkNotNull(args); String volumeName = args.getVolumeName(); String bucketName = args.getBucketName(); - String keyName = args.getKeyName(); + String keyName = OMClientRequest.validateAndNormalizeKey( + enableFileSystemPaths, args.getKeyName()); + + metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, bucketName); OmKeyInfo value = null; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 13e47a7cb087..f70bf33cf975 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2226,7 +2226,6 @@ public OmKeyInfo lookupKey(OmKeyArgs args) throws IOException { Map auditMap = bucket.audit(args.toAuditMap()); args = bucket.update(args); - try { metrics.incNumKeyLookups(); return keyManager.lookupKey(args, getClientAddress()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java index 0fa9ca1a8d2c..f0b8cf5bfc7d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java @@ -295,7 +295,7 @@ public static String validateAndNormalizeKey(String keyName) .normalize().getPath(); } if (!keyName.equals(normalizedKeyName)) { - LOG.debug("Normalized key {} to {} ", keyName, + LOG.info("Normalized key {} to {} ", keyName, normalizedKeyName.substring(1)); } return isValidKeyPath(normalizedKeyName.substring(1)); From bf4456b4aa8c43052e1740e14d0b800198317289 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Tue, 11 Aug 2020 11:05:36 -0700 Subject: [PATCH 2/2] remove not required --- .../src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java | 1 + .../org/apache/hadoop/ozone/om/request/OMClientRequest.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index f70bf33cf975..13e47a7cb087 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2226,6 +2226,7 @@ public OmKeyInfo lookupKey(OmKeyArgs args) throws IOException { Map auditMap = bucket.audit(args.toAuditMap()); args = bucket.update(args); + try { metrics.incNumKeyLookups(); return keyManager.lookupKey(args, getClientAddress()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java index f0b8cf5bfc7d..0fa9ca1a8d2c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java @@ -295,7 +295,7 @@ public static String validateAndNormalizeKey(String keyName) .normalize().getPath(); } if (!keyName.equals(normalizedKeyName)) { - LOG.info("Normalized key {} to {} ", keyName, + LOG.debug("Normalized key {} to {} ", keyName, normalizedKeyName.substring(1)); } return isValidKeyPath(normalizedKeyName.substring(1));