apache · smengcl · Jun 15, 2020 · Jun 18, 2020 · Jun 18, 2020 · Jun 19, 2020
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -81,7 +81,9 @@ protected ObjectStore() {
     proxy = null;
   }
 
-  @VisibleForTesting
+  /**
+   * Returns the ClientProtocol of the ObjectStore.
+   */
   public ClientProtocol getClientProxy() {
     return proxy;
   }

diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+CORE-SITE.XML_fs.defaultFS=ofs://om
+
 OZONE-SITE.XML_ozone.csi.owner=hadoop
 OZONE-SITE.XML_ozone.csi.socket=/tmp/csi.sock
 

diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-config
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
+CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs
 MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar
 
 no_proxy=om,scm,s3g,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-config
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
+CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs
 MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar
 
 no_proxy=om,scm,s3g,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-config
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
+CORE-SITE.xml_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs
 MAPRED-SITE.XML_mapreduce.application.classpath=/opt/hadoop/share/hadoop/mapreduce/*:/opt/hadoop/share/hadoop/mapreduce/lib/*:/opt/ozone/share/ozone/lib/[email protected]@.jar
 
 no_proxy=om,scm,s3g,kdc,localhost,127.0.0.1
diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config
@@ -14,8 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
-CORE-SITE.XML_fs.defaultFS=o3fs://bucket.volume.id1
+CORE-SITE.XML_fs.defaultFS=ofs://id1
+
 OZONE-SITE.XML_ozone.om.service.ids=id1
 OZONE-SITE.XML_ozone.om.nodes.id1=om1,om2,om3
 OZONE-SITE.XML_ozone.om.address.id1.om1=om1

diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config
@@ -14,8 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
-CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.volume1.omservice
+CORE-SITE.XML_fs.defaultFS=ofs://omservice
+
 OZONE-SITE.XML_ozone.om.service.ids=omservice
 OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3
 OZONE-SITE.XML_ozone.om.address.omservice.om1=om1

diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+CORE-SITE.XML_fs.defaultFS=ofs://om
+
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.container.size=256MB

diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config
@@ -14,8 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem
-CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
+CORE-SITE.XML_fs.defaultFS=ofs://om
+
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.container.size=1GB

diff --git a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+CORE-SITE.XML_fs.defaultFS=ofs://om
+
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
 OZONE-SITE.XML_ozone.scm.names=scm

diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config
@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+CORE-SITE.XML_fs.defaultFS=ofs://om
+
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.scm.container.size=1GB
 OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
@@ -67,7 +67,8 @@ HDFS-SITE.XML_rpc.metrics.quantile.enable=true
 HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
 
 CORE-SITE.XML_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs
-CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.volume1/
+CORE-SITE.XML_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs
+CORE-SITE.XML_fs.defaultFS=ofs://om
 
 MAPRED-SITE.XML_mapreduce.framework.name=yarn
 MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config
@@ -14,8 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
-CORE-SITE.XML_fs.defaultFS=o3fs://bucket.volume.id1
+CORE-SITE.XML_fs.defaultFS=ofs://id1
+
 OZONE-SITE.XML_ozone.om.service.ids=id1
 OZONE-SITE.XML_ozone.om.internal.service.id=id1
 OZONE-SITE.XML_ozone.om.nodes.id1=om1,om2,om3

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -14,12 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem
-CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
-OZONE-SITE.XML_ozone.om.volume.listall.allowed=false
+CORE-SITE.XML_fs.defaultFS=ofs://om
 
 OZONE-SITE.XML_ozone.om.address=om
 OZONE-SITE.XML_ozone.om.http-address=om:9874
+OZONE-SITE.XML_ozone.om.volume.listall.allowed=false
 OZONE-SITE.XML_ozone.scm.container.size=1GB
 OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
 OZONE-SITE.XML_ozone.scm.names=scm
@@ -89,7 +88,6 @@ CORE-SITE.XML_hadoop.http.authentication.type=kerberos
 CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/[email protected]
 CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
 
-
 CORE-SITE.XML_hadoop.security.authorization=true
 HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=*
 HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=*

diff --git a/...est/mini-chaos-tests/src/test/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/...est/mini-chaos-tests/src/test/resources/META-INF/services/org.apache.hadoop.fs.FileSystem
@@ -14,3 +14,4 @@
 # limitations under the License.
 
 org.apache.hadoop.fs.ozone.OzoneFileSystem
+org.apache.hadoop.fs.ozone.RootedOzoneFileSystem
diff --git a/.../integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java b/.../integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -82,7 +82,7 @@ public class TestRootedOzoneFileSystem {
   private FileSystem fs;
   private RootedOzoneFileSystem ofs;
   private ObjectStore objectStore;
-  private static BasicRootedOzoneClientAdapterImpl adapter;
+  private static BasicRootedOzoneFileSystemImpl impl;
 
   private String volumeName;
   private String bucketName;
@@ -112,12 +112,10 @@ public void init() throws Exception {
 
     // Set the fs.defaultFS and start the filesystem
     conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
-    // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
-    //  hence this workaround.
-    conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+    // fs.ofs.impl should be loaded from META-INF, no need to explicitly set it
     fs = FileSystem.get(conf);
     ofs = (RootedOzoneFileSystem) fs;
-    adapter = (BasicRootedOzoneClientAdapterImpl) ofs.getAdapter();
+    impl = ofs.getImpl();
   }
 
   @After
@@ -131,10 +129,7 @@ public void teardown() {
   @Test
   public void testOzoneFsServiceLoader() throws IOException {
     OzoneConfiguration confTestLoader = new OzoneConfiguration();
-    // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
-    //  hence this workaround.
-    confTestLoader.set("fs.ofs.impl",
-        "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+    // fs.ofs.impl should be loaded from META-INF, no need to explicitly set it
     Assert.assertEquals(FileSystem.getFileSystemClass(
         OzoneConsts.OZONE_OFS_URI_SCHEME, confTestLoader),
         RootedOzoneFileSystem.class);
@@ -586,7 +581,7 @@ private void listStatusRecursiveHelper(Path curPath, List<FileStatus> result)
    */
   private List<FileStatus> callAdapterListStatus(String pathStr,
       boolean recursive, String startPath, long numEntries) throws IOException {
-    return adapter.listStatus(pathStr, recursive, startPath, numEntries,
+    return impl.listStatus(pathStr, recursive, startPath, numEntries,
         ofs.getUri(), ofs.getWorkingDirectory(), ofs.getUsername())
         .stream().map(ofs::convertFileStatus).collect(Collectors.toList());
   }

diff --git a/...on-test/src/test/java/org/apache/hadoop/fs/ozone/contract/rooted/RootedOzoneContract.java b/...on-test/src/test/java/org/apache/hadoop/fs/ozone/contract/rooted/RootedOzoneContract.java
@@ -104,12 +104,7 @@ public FileSystem getTestFileSystem() throws IOException {
         OzoneConsts.OZONE_OFS_URI_SCHEME,
         cluster.getOzoneManager().getRpcPort());
     getConf().set("fs.defaultFS", uri);
-
-    // Note: FileSystem#loadFileSystems doesn't load OFS class because
-    //  META-INF points to org.apache.hadoop.fs.ozone.OzoneFileSystem
-    getConf().set("fs.ofs.impl",
-        "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
-
+    // fs.ofs.impl should be loaded from META-INF, no need to explicitly set it
     copyClusterConfigs(OMConfigKeys.OZONE_OM_ADDRESS_KEY);
     copyClusterConfigs(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
     return FileSystem.get(getConf());

diff --git a/...-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java b/...-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -460,8 +460,7 @@ private OzoneConfiguration getClientConfForOFS(
       String hostPrefix, OzoneConfiguration configuration) {
 
     OzoneConfiguration clientConf = new OzoneConfiguration(configuration);
-    clientConf.set("fs.ofs.impl",
-        "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+    // fs.ofs.impl should be loaded from META-INF, no need to explicitly set it
     clientConf.set(FS_DEFAULT_NAME_KEY, hostPrefix);
     clientConf.setInt(FS_TRASH_INTERVAL_KEY, 60);
     return clientConf;

diff --git a/.../ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/.../ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@ -228,11 +228,6 @@ public void renameKey(String key, String newKeyName) throws IOException {
     bucket.renameKey(key, newKeyName);
   }
 
-  @Override
-  public void rename(String pathStr, String newPath) throws IOException {
-    throw new IOException("Please use renameKey instead for o3fs.");
-  }
-
   /**
    * Helper method to create an directory specified by key name in bucket.
    *
Original file line number	Diff line number	Diff line change
Expand Up		@@ -14,3 +14,4 @@
		# limitations under the License.

		org.apache.hadoop.fs.ozone.OzoneFileSystem
		org.apache.hadoop.fs.ozone.RootedOzoneFileSystem