HDDS-3821. Disable Ozone SPNEGO should not fall back to hadoop.http.a…

hadoop-hdds/docs/content/security/SecureOzone.md

@@ -100,12 +100,12 @@ All these settings should be made in ozone-site.xml.
             <td>The keytab file used by SCM daemon to login as its service principal.</td>
           </tr>
           <tr>
-            <td>hdds.scm.http.kerberos.principal</th>
-            <td>SCM http server service principal.</td>
+            <td>hdds.scm.http.auth.kerberos.principal</th>
+            <td>SCM http server service principal if SPNEGO is enabled for SCM http server.</td>
           </tr>
           <tr>
-            <td>hdds.scm.http.kerberos.keytab</th>
-            <td>The keytab file used by SCM http server to login as its service principal.</td>
+            <td>hdds.scm.http.auth.kerberos.keytab</th>
+            <td>The keytab file used by SCM http server to login as its service principal if SPNEGO is enabled for SCM http server</td>
           </tr>
         </tbody>
       </table>
@@ -136,12 +136,12 @@ All these settings should be made in ozone-site.xml.
             <td>TThe keytab file used by SCM daemon to login as its service principal.</td>
           </tr>
           <tr>
-            <td>ozone.om.http.kerberos.principal</th>
-            <td>Ozone Manager http server service principal.</td>
+            <td>ozone.om.http.auth.kerberos.principal</th>
+            <td>Ozone Manager http server service principal if SPNEGO is enabled for om http server.</td>
           </tr>
           <tr>
-            <td>ozone.om.http.kerberos.keytab</th>
-            <td>The keytab file used by OM http server to login as its service principal.</td>
+            <td>ozone.om.http.auth.kerberos.keytab</th>
+            <td>The keytab file used by OM http server to login as its service principal if SPNEGO is enabled for om http server.</td>
           </tr>
         </tbody>
       </table>
@@ -165,11 +165,11 @@ All these settings should be made in ozone-site.xml.
         <tbody>
           <tr>
             <td>ozone.s3g.http.auth.kerberos.principal</th>
-            <td>S3 Gateway principal. <br/> e.g. HTTP/[email protected]</td>
+            <td>S3 Gateway principal if SPNEGO is enabled for S3 Gateway http server. <br/> e.g. HTTP/[email protected]</td>
           </tr>
           <tr>
             <td>ozone.s3g.http.auth.kerberos.keytab</th>
-            <td>The keytab file used by S3 gateway</td>
+            <td>The keytab file used by S3 gateway if SPNEGO is enabled for S3 Gateway http server.</td>
           </tr>
         </tbody>
       </table>

hadoop-hdds/docs/content/security/SecuringOzoneHTTP.md

@@ -0,0 +1,141 @@
+---
+title: "Securing Ozone HTTP web-consoles"
+date: "2020-June-17"
+summary: Secure HTTP web-consoles for Ozone services 
+weight: 3
+icon: lock
+---
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+This document describes how to configure Ozone HTTP web-consoles to require user authentication. 
+
+### Default authentication 
+
+By default Ozone HTTP web-consoles (OM, SCM, S3G, Recon, Datanode)
+allow access without authentication based on the following default configurations. 
+
+Property| Value
+-----------------------------------|-----------------------------------------
+ozone.security.http.kerberos.enabled | false
+ozone.http.filter.initializers | <empty>
+
+If you have an SPNEGO enabled Ozone cluster and want to disable it for all Ozone services,
+just make sure the two key mentioned are configured as above.
+
+### Kerberos based SPNEGO authentication
+However, they can be configured to require Kerberos authentication using HTTP SPNEGO protocol (supported
+by browsers like Firefox and Chrome). To achieve that, the following keys must
+be configured first. 
+
+Property| Value
+-----------------------------------|-----------------------------------------
+hadoop.security.authentication | kerberos
+ozone.security.http.kerberos.enabled | true
+ozone.http.filter.initializers | org.apache.hadoop.security.AuthenticationFilterInitializer
+
+After that, individual component needs to configure properly to completely enable
+SPNEGO or SIMPLE authentication.
+
+### Enable SPNEGO authentication for OM HTTP
+Property| Value
+-----------------------------------|-----------------------------------------
+ozone.om.http.auth.type | kerberos
+ozone.om.http.auth.kerberos.principal | HTTP/_HOST@REALM
+ozone.om.http.auth.kerberos.keytab| /path/to/HTTP.keytab
+
+### Enable SPNEGO authentication for S3G HTTP
+Property| Value
+-----------------------------------|-----------------------------------------
+ozone.s3g.http.auth.type | kerberos
+ozone.s3g.http.auth.kerberos.principal | HTTP/_HOST@REALM
+ozone.s3g.http.auth.kerberos.keytab| /path/to/HTTP.keytab
+
+### Enable SPNEGO authentication for RECON HTTP
+Property| Value
+-----------------------------------|-----------------------------------------
+ozone.recon.http.auth.type | kerberos
+ozone.recon.http.auth.kerberos.principal | HTTP/_HOST@REALM
+ozone.recon.http.auth.kerberos.keytab| /path/to/HTTP.keytab
+
+### Enable SPNEGO authentication for SCM HTTP
+Property| Value
+-----------------------------------|-----------------------------------------
+hdds.scm.http.auth.type | kerberos
+hdds.scm.http.auth.kerberos.principal | HTTP/_HOST@REALM
+hdds.scm.http.auth.kerberos.keytab| /path/to/HTTP.keytab
+
+### Enable SPNEGO authentication for DATANODE HTTP
+Property| Value
+-----------------------------------|-----------------------------------------
+hdds.datanode.http.auth.type | kerberos
+hdds.datanode.http.auth.kerberos.principal | HTTP/_HOST@REALM
+hdds.datanode.http.auth.kerberos.keytab| /path/to/HTTP.keytab
+
+Note: Ozone datanode does not have a default webpage, which prevents you from 
+accessing "/" or "/index.html". But it does provide standard 
+servlet like jmx/conf/jstack via HTTP. 
+
+In addition, Ozone HTTP web-console support the equivalent of Hadoop's Pseudo/Simple authentication. 
+If this option is enabled, the user name must be specified in the first browser interaction using the user.name 
+query string parameter. e.g., http://scm:9876/?user.name=scmadmin.
+
+### Enable SIMPLE authentication for OM HTTP
+Property| Value
+-----------------------------------|-----------------------------------------
+ozone.om.http.auth.type | simple
+ozone.om.http.auth.simple.anonymous_allowed | false
+
+If you don't want to specify the user.name in the query string parameter, 
+change ozone.om.http.auth.simple.anonymous_allowed to true.
+
+### Enable SIMPLE authentication for S3G HTTP
+Property| Value
+-----------------------------------|-----------------------------------------
+ozone.s3g.http.auth.type | simple
+ozone.s3g.http.auth.simple.anonymous_allowed | false
+
+If you don't want to specify the user.name in the query string parameter, 
+change ozone.s3g.http.auth.simple.anonymous_allowed to true.
+
+### Enable SIMPLE authentication for RECON HTTP
+Property| Value
+-----------------------------------|-----------------------------------------
+ozone.recon.http.auth.type | simple
+ozone.recon.http.auth.simple.anonymous_allowed | false
+
+If you don't want to specify the user.name in the query string parameter, 
+change ozone.recon.http.auth.simple.anonymous_allowed to true.
+
+
+### Enable SIMPLE authentication for SCM HTTP
+Property| Value
+-----------------------------------|-----------------------------------------
+ozone.scm.http.auth.type | simple
+ozone.scm.http.auth.simple.anonymous_allowed | false
+
+If you don't want to specify the user.name in the query string parameter, 
+change ozone.scm.http.auth.simple.anonymous_allowed to true.
+
+### Enable SIMPLE authentication for DATANODE HTTP
+Property| Value
+-----------------------------------|-----------------------------------------
+ozone.datanode.http.auth.type | simple
+ozone.datanode.http.auth.simple.anonymous_allowed | false
+
+If you don't want to specify the user.name in the query string parameter, 
+change ozone.datanode.http.auth.simple.anonymous_allowed to true.

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java

@@ -115,9 +115,11 @@ public BaseHttpServer(ConfigurationSource conf, String name)
       if (isSecurityEnabled) {
         String httpAuthType = conf.get(getHttpAuthType(), "simple");
         LOG.info("HttpAuthType: {} = {}", getHttpAuthType(), httpAuthType);
+        // Ozone config prefix must be set to avoid AuthenticationFilter
+        // fall back to default one form hadoop.http.authentication.
+        builder.authFilterConfigurationPrefix(getHttpAuthConfigPrefix());
         if (httpAuthType.equals("kerberos")) {
           builder.setSecurityEnabled(true);
-          builder.authFilterConfigurationPrefix(getHttpAuthConfigPrefix());
           builder.setUsernameConfKey(getSpnegoPrincipal());
           builder.setKeytabConfKey(getKeytabFile());
         }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java

@@ -572,12 +572,13 @@ private HttpServer2(final Builder b) throws IOException {
     this.findPort = b.findPort;
     this.portRanges = b.portRanges;
     initializeWebServer(b.name, b.hostName, b.conf, b.pathSpecs,
-        b.authFilterConfigurationPrefix);
+        b.authFilterConfigurationPrefix, b.securityEnabled);
   }
 
   private void initializeWebServer(String name, String hostName,
       ConfigurationSource conf, String[] pathSpecs,
-      String authFilterConfigPrefix) throws IOException {
+      String authFilterConfigPrefix,
+      boolean securityEnabled) throws IOException {
 
     Preconditions.checkNotNull(webAppContext);
 
@@ -614,14 +615,17 @@ private void initializeWebServer(String name, String hostName,
     final FilterInitializer[] initializers = getFilterInitializers(conf);
     if (initializers != null) {
       conf.set(BIND_ADDRESS, hostName);
+      org.apache.hadoop.conf.Configuration hadoopConf =
+          LegacyHadoopConfigurationSource.asHadoopConfiguration(conf);
+      Map<String, String> filterConfig = getFilterConfigMap(hadoopConf,
+          authFilterConfigPrefix);
       for (FilterInitializer c : initializers) {
-        //c.initFilter(this, conf) does not work here as it does not take config
-        // prefix key.
-        Map<String, String> filterConfig = getFilterConfigMap(
-            LegacyHadoopConfigurationSource.asHadoopConfiguration(conf),
-            authFilterConfigPrefix);
-        addFilter("authentication", AuthenticationFilter.class.getName(),
-            filterConfig);
+        if ((c instanceof AuthenticationFilterInitializer) && securityEnabled) {
+          addFilter("authentication",
+              AuthenticationFilter.class.getName(), filterConfig);
+        } else {
+          c.initFilter(this, hadoopConf);
+        }
       }
     }
 

hadoop-ozone/integration-test/src/test/resources/log4j.properties

@@ -19,3 +19,6 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%
 
 log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
 log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+
+log4j.logger.org.apache.ratis.grpc.server.GrpcLogAppender=DEBUG
+log4j.logger.org.apache.ratis.server.impl.RaftServerImpl=DEBUG