From 335cc58fb16e52ed407178c491f6d115d13823c1 Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Fri, 28 Aug 2020 09:15:28 -0700 Subject: [PATCH 1/4] Revert "Revert docker-config fs.defaultFS changes." This reverts commit 0e5ed8004e4c84e4b21fb9a8ff636e8802674ffe. --- hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config | 2 +- hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config | 2 +- hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config | 2 +- .../dist/src/main/compose/ozonesecure-om-ha/docker-config | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config index 4d47bf0b6f19..863251e13396 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -CORE-SITE.XML_fs.defaultFS=o3fs://bucket.volume.id1 +CORE-SITE.XML_fs.defaultFS=ofs://id1 OZONE-SITE.XML_ozone.om.service.ids=id1 OZONE-SITE.XML_ozone.om.nodes.id1=om1,om2,om3 OZONE-SITE.XML_ozone.om.address.id1.om1=om1 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config index 008c684aeb99..615c71d6d1b0 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.volume1.omservice +CORE-SITE.XML_fs.defaultFS=ofs://omservice OZONE-SITE.XML_ozone.om.service.ids=omservice OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3 OZONE-SITE.XML_ozone.om.address.omservice.om1=om1 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config index 7f1050db8b90..2868bdf4046c 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config @@ -68,7 +68,7 @@ HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 CORE-SITE.XML_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs CORE-SITE.XML_fs.AbstractFileSystem.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzFs -CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.volume1/ +CORE-SITE.XML_fs.defaultFS=ofs://om MAPRED-SITE.XML_mapreduce.framework.name=yarn MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config index c332448a9508..a67522059411 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -CORE-SITE.XML_fs.defaultFS=o3fs://bucket.volume.id1 +CORE-SITE.XML_fs.defaultFS=ofs://id1 OZONE-SITE.XML_ozone.om.service.ids=id1 OZONE-SITE.XML_ozone.om.internal.service.id=id1 OZONE-SITE.XML_ozone.om.nodes.id1=om1,om2,om3 From ae117feeb122dc1cff39325dd93241f12b4f3c57 Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Thu, 18 Jun 2020 14:22:47 -0700 Subject: [PATCH 2/4] Set fs.defaultFS to OFS in docker-config. (cherry picked from commit ba080fc9e6ef4391864417ac1daf0d2953712aae) (cherry picked from commit 4593e8ae425db0c59eaaed04ebb64ef9f8648ce5) --- hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config | 2 ++ .../dist/src/main/compose/ozone-om-ha-s3/docker-config | 1 + hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config | 1 + .../dist/src/main/compose/ozone-topology/docker-config | 2 ++ hadoop-ozone/dist/src/main/compose/ozone/docker-config | 2 ++ .../dist/src/main/compose/ozoneblockade/docker-config | 2 ++ .../dist/src/main/compose/ozones3-haproxy/docker-config | 2 ++ .../dist/src/main/compose/ozonesecure-om-ha/docker-config | 1 + hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config | 4 ++-- 9 files changed, 15 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config index 2f48fed77a79..179f03a6c5b5 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-csi/docker-config @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +CORE-SITE.XML_fs.defaultFS=ofs://om + OZONE-SITE.XML_ozone.csi.owner=hadoop OZONE-SITE.XML_ozone.csi.socket=/tmp/csi.sock diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config index 863251e13396..bc74641320ec 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha-s3/docker-config @@ -15,6 +15,7 @@ # limitations under the License. CORE-SITE.XML_fs.defaultFS=ofs://id1 + OZONE-SITE.XML_ozone.om.service.ids=id1 OZONE-SITE.XML_ozone.om.nodes.id1=om1,om2,om3 OZONE-SITE.XML_ozone.om.address.id1.om1=om1 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config index 615c71d6d1b0..805f9f30f218 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config @@ -15,6 +15,7 @@ # limitations under the License. CORE-SITE.XML_fs.defaultFS=ofs://omservice + OZONE-SITE.XML_ozone.om.service.ids=omservice OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3 OZONE-SITE.XML_ozone.om.address.omservice.om1=om1 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config index 9cf5822cc73d..648a858aa1c7 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-topology/docker-config @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +CORE-SITE.XML_fs.defaultFS=ofs://om + OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.container.size=256MB diff --git a/hadoop-ozone/dist/src/main/compose/ozone/docker-config b/hadoop-ozone/dist/src/main/compose/ozone/docker-config index f100e823517a..a16031160d9f 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone/docker-config @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +CORE-SITE.XML_fs.defaultFS=ofs://om + OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.container.size=1GB diff --git a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config index 8e4f0ae9b8eb..be364ba6ec36 100644 --- a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +CORE-SITE.XML_fs.defaultFS=ofs://om + OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 OZONE-SITE.XML_ozone.scm.names=scm diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config index 71b03332d3bf..485fe9c1bce6 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +CORE-SITE.XML_fs.defaultFS=ofs://om + OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config index a67522059411..00820d3d72cd 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-om-ha/docker-config @@ -15,6 +15,7 @@ # limitations under the License. CORE-SITE.XML_fs.defaultFS=ofs://id1 + OZONE-SITE.XML_ozone.om.service.ids=id1 OZONE-SITE.XML_ozone.om.internal.service.id=id1 OZONE-SITE.XML_ozone.om.nodes.id1=om1,om2,om3 diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config index 30e1816ab8c1..3ac565d276b3 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config @@ -14,10 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -OZONE-SITE.XML_ozone.om.volume.listall.allowed=false +CORE-SITE.XML_fs.defaultFS=ofs://om OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 +OZONE-SITE.XML_ozone.om.volume.listall.allowed=false OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.names=scm @@ -87,7 +88,6 @@ CORE-SITE.XML_hadoop.http.authentication.type=kerberos CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab - CORE-SITE.XML_hadoop.security.authorization=true HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=* HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=* From de624261eee357725a749fbfa9eba904e9df02ff Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Fri, 28 Aug 2020 09:20:33 -0700 Subject: [PATCH 3/4] Remove unnecessary changes. Change-Id: I06f1568e821b2d8b902a3c4d20d3c92372e4976d --- hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config index 3ac565d276b3..415efad6122b 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config @@ -18,7 +18,6 @@ CORE-SITE.XML_fs.defaultFS=ofs://om OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 -OZONE-SITE.XML_ozone.om.volume.listall.allowed=false OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1 OZONE-SITE.XML_ozone.scm.names=scm @@ -88,6 +87,7 @@ CORE-SITE.XML_hadoop.http.authentication.type=kerberos CORE-SITE.XML_hadoop.http.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM CORE-SITE.XML_hadoop.http.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab + CORE-SITE.XML_hadoop.security.authorization=true HADOOP-POLICY.XML_ozone.om.security.client.protocol.acl=* HADOOP-POLICY.XML_hdds.security.client.datanode.container.protocol.acl=* From eac114a22ad1c0bc40c93b0f1fc1182c7e4bd28b Mon Sep 17 00:00:00 2001 From: Siyao Meng Date: Sat, 5 Sep 2020 11:05:35 -0700 Subject: [PATCH 4/4] Update more docker fs.defaultFS --- hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config | 2 +- hadoop-ozone/dist/src/main/compose/ozone-mr/common-config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config index d378a679f816..95f840dbe819 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/docker-config @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -CORE-SITE.XML_fs.defaultFS=o3fs://bucket1.volume1.omservice +CORE-SITE.XML_fs.defaultFS=ofs://omservice/ OZONE-SITE.XML_ozone.om.service.ids=omservice OZONE-SITE.XML_ozone.om.nodes.omservice=om1,om2,om3 diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config index 46e7527315cc..6e60c5586b21 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/common-config @@ -30,7 +30,7 @@ OZONE-SITE.XML_hdds.datanode.dir=/data/hdds HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019 HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012 -CORE-SITE.xml_fs.defaultFS=o3fs://bucket1.volume1/ +CORE-SITE.xml_fs.defaultFS=ofs://om/ MAPRED-SITE.XML_mapreduce.framework.name=yarn MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME