AMBARI-22903. Log Search: Add Knox to docker dev env

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/auth/filter/AbstractJWTFilter.java

@@ -18,6 +18,7 @@
  */
 package org.apache.ambari.logsearch.auth.filter;
 
+import com.google.gson.Gson;
 import io.jsonwebtoken.Claims;
 import io.jsonwebtoken.ExpiredJwtException;
 import io.jsonwebtoken.Jwts;
@@ -46,12 +47,15 @@
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
 import java.security.cert.CertificateException;
 import java.security.cert.CertificateFactory;
 import java.security.cert.X509Certificate;
 import java.security.interfaces.RSAPublicKey;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 public abstract class AbstractJWTFilter extends AbstractAuthenticationProcessingFilter {
 
@@ -110,14 +114,27 @@ public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain)
   @Override
   protected void successfulAuthentication(HttpServletRequest request, HttpServletResponse response, FilterChain chain, Authentication authResult) throws IOException, ServletException {
     super.successfulAuthentication(request, response, chain, authResult);
-    response.sendRedirect(request.getRequestURL().toString() + getOriginalQueryString(request));
+    String ajaxRequestHeader = request.getHeader("X-Requested-With");
+    if (isWebUserAgent(request.getHeader("User-Agent")) && !"XMLHttpRequest".equals(ajaxRequestHeader)) {
+      response.sendRedirect(request.getRequestURL().toString() + getOriginalQueryString(request));
+    }
+    // chain.doFilter(request, response); TODO: check
   }
 
   @Override
   protected void unsuccessfulAuthentication(HttpServletRequest request, HttpServletResponse response, AuthenticationException failed) throws IOException, ServletException {
     super.unsuccessfulAuthentication(request, response, failed);
+    String ajaxRequestHeader = request.getHeader("X-Requested-With");
     String loginUrl = constructLoginURL(request);
-    response.sendRedirect(loginUrl);
+    if (!isWebUserAgent(request.getHeader("User-Agent")) || "XMLHttpRequest".equals(ajaxRequestHeader)) {
+      Map<String, String> mapObj = new HashMap<>();
+      mapObj.put("knoxssoredirectURL", URLEncoder.encode(loginUrl, "UTF-8"));
+      response.setContentType("application/json");
+      response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+      response.sendError(HttpServletResponse.SC_UNAUTHORIZED,  new Gson().toJson(mapObj));
+    } else {
+      response.sendRedirect(loginUrl);
+    }
   }
 
   private String getJWTFromCookie(HttpServletRequest req) {
@@ -135,6 +152,20 @@ private String getJWTFromCookie(HttpServletRequest req) {
     return serializedJWT;
   }
 
+  private boolean isWebUserAgent(String userAgent) {
+    boolean isWeb = false;
+    List<String> userAgentList = getUserAgentList();
+    if (userAgentList != null && userAgentList.size() > 0) {
+      for (String ua : userAgentList) {
+        if (StringUtils.startsWithIgnoreCase(userAgent, ua)) {
+          isWeb = true;
+          break;
+        }
+      }
+    }
+    return isWeb;
+  }
+
   private RSAPublicKey parseRSAPublicKey(String pem) throws ServletException {
     String fullPem = PEM_HEADER + pem + PEM_FOOTER;
     try {
@@ -190,4 +221,6 @@ private boolean isAuthenticated(Authentication authentication) {
 
   protected abstract Collection<? extends GrantedAuthority> getAuthorities();
 
+  protected abstract List<String> getUserAgentList();
+
 }

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/AuthPropsConfig.java

@@ -137,7 +137,7 @@ public class AuthPropsConfig {
     sources = {LOGSEARCH_PROPERTIES_FILE}
   )
   private String cookieName;
-  @Value("${logsearch.auth.jwt.query.param.original_url:originalUrl=}")
+  @Value("${logsearch.auth.jwt.query.param.original_url:originalUrl}")
   @LogSearchPropertyDescription(
     name = "logsearch.auth.jwt.query.param.original_url",
     description = "Name of the original request URL which is used to redirect to Log Search Portal.",
@@ -157,6 +157,16 @@ public class AuthPropsConfig {
   )
   private List<String> audiences;
 
+  @Value("#{'${logsearch.auth.jwt.user.agents:Mozilla,Opera,Chrome}'.split(',')}")
+  @LogSearchPropertyDescription(
+    name = "logsearch.auth.jwt.user.agents",
+    description = "Comma separated web user agent list. (Used as prefixes)",
+    examples = {"Mozilla,Chrome"},
+    defaultValue = "Mozilla,Opera,Chrome",
+    sources = {LOGSEARCH_PROPERTIES_FILE}
+  )
+  private List<String> userAgentList;
+
   @Value("#{'${logsearch.roles.allowed:AMBARI.ADMINISTRATOR,CLUSTER.ADMINISTRATOR}'.split(',')}")
   @LogSearchPropertyDescription(
     name = "logsearch.roles.allowed",
@@ -296,4 +306,12 @@ public boolean isRedirectForward() {
   public void setRedirectForward(boolean redirectForward) {
     this.redirectForward = redirectForward;
   }
+
+  public List<String> getUserAgentList() {
+    return this.userAgentList;
+  }
+
+  public void setUserAgentList(List<String> userAgentList) {
+    this.userAgentList = userAgentList;
+  }
 }

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java

@@ -51,6 +51,7 @@
 
 import javax.inject.Inject;
 import javax.inject.Named;
+import java.util.ArrayList;
 import java.util.List;
 
 import static org.apache.ambari.logsearch.common.LogSearchConstants.LOGSEARCH_SESSION_ID;
@@ -120,7 +121,7 @@ protected void configure(HttpSecurity http) throws Exception {
       .addFilterBefore(logsearchJwtFilter(), LogsearchSecurityContextFormationFilter.class)
       .logout()
         .logoutUrl("/logout")
-        .deleteCookies(LOGSEARCH_SESSION_ID)
+        .deleteCookies(getCookies())
         .logoutSuccessHandler(new LogsearchLogoutSuccessHandler());
   }
 
@@ -196,7 +197,9 @@ public RequestMatcher requestMatcher() {
     matchers.add(new AntPathRequestMatcher("/docs/**"));
     matchers.add(new AntPathRequestMatcher("/swagger-ui/**"));
     matchers.add(new AntPathRequestMatcher("/swagger.html"));
-    matchers.add(new AntPathRequestMatcher("/"));
+    if (!authPropsConfig.isAuthJwtEnabled()) {
+      matchers.add(new AntPathRequestMatcher("/"));
+    }
     matchers.add(new AntPathRequestMatcher("/login"));
     matchers.add(new AntPathRequestMatcher("/logout"));
     matchers.add(new AntPathRequestMatcher("/resources/**"));
@@ -205,7 +208,6 @@ public RequestMatcher requestMatcher() {
     matchers.add(new AntPathRequestMatcher("/assets/**"));
     matchers.add(new AntPathRequestMatcher("/templates/**"));
     matchers.add(new AntPathRequestMatcher("/api/v1/info/**"));
-    matchers.add(new AntPathRequestMatcher("/api/v1/public/**"));
     matchers.add(new AntPathRequestMatcher("/api/v1/swagger.json"));
     matchers.add(new AntPathRequestMatcher("/api/v1/swagger.yaml"));
     return new OrRequestMatcher(matchers);
@@ -227,4 +229,13 @@ public RequestMatcher logsearchConfigRequestMatcher() {
     return new AntPathRequestMatcher("/api/v1/shipper/**");
   }
 
+  private String[] getCookies() {
+    List<String> cookies = new ArrayList<>();
+    cookies.add(LOGSEARCH_SESSION_ID);
+    if (authPropsConfig.isAuthJwtEnabled()) {
+      cookies.add(authPropsConfig.getCookieName());
+    }
+    return cookies.toArray(new String[0]);
+  }
+
 }

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchJWTFilter.java

@@ -66,6 +66,11 @@ protected List<String> getAudiences() {
     return authPropsConfig.getAudiences();
   }
 
+  @Override
+  protected List<String> getUserAgentList() {
+    return authPropsConfig.getUserAgentList();
+  }
+
   @Override
   protected Collection<? extends GrantedAuthority> getAuthorities() {
     return null; // TODO

ambari-logsearch/docker/Dockerfile

@@ -57,6 +57,25 @@ RUN git config --global url."https://".insteadOf git://
 ENV SOLR_VERSION 6.6.2
 RUN wget --no-check-certificate -O /root/solr-$SOLR_VERSION.tgz http://public-repo-1.hortonworks.com/ARTIFACTS/dist/lucene/solr/$SOLR_VERSION/solr-$SOLR_VERSION.tgz
 RUN cd /root && tar -zxvf /root/solr-$SOLR_VERSION.tgz
+
+# Install Knox
+WORKDIR /
+RUN adduser knox
+ENV KNOX_VERSION 0.14.0
+RUN wget -q -O /knox-${KNOX_VERSION}.zip http://download.nextag.com/apache/knox/${KNOX_VERSION}/knox-${KNOX_VERSION}.zip && unzip /knox-${KNOX_VERSION}.zip && rm knox-${KNOX_VERSION}.zip && ln -nsf knox-${KNOX_VERSION} knox && chmod +x /knox/bin/*.sh && chown -R knox /knox/
+
+ADD knox/keystores /knox-secrets
+RUN cd /knox-secrets && unzip test-secrets.zip
+RUN mkdir -p /knox/data/security/keystores
+RUN mv /knox-secrets/master /knox/data/security/master
+RUN cd /knox-secrets && cp -r * /knox/data/security/keystores/
+RUN chown -R knox /knox/data/security
+
+ADD knox/ldap.sh /ldap.sh
+ADD knox/gateway.sh /gateway.sh
+RUN touch /gateway.out && chown -R knox /gateway.out
+RUN touch /ldap.out && chown -R knox /ldap.out
+
 ADD bin/start.sh /root/start.sh
 ADD test-config /root/test-config
 ADD test-logs /root/test-logs

ambari-logsearch/docker/bin/start.sh

@@ -50,12 +50,18 @@ function create_logsearch_configs() {
   cp /root/test-config/logsearch/log4j.xml /root/config/logsearch/
   cp /root/test-config/logsearch/logsearch-env.sh /root/config/logsearch/
   cp $LOGSEARCH_SERVER_PATH/conf/user_pass.json /root/config/logsearch/user_pass.json
-  if [ $LOGSEARCH_HTTPS_ENABLED == 'true' ]
+  if [ "$LOGSEARCH_HTTPS_ENABLED" == "true" ]
   then
     cp /root/test-config/logsearch/logsearch-https.properties /root/config/logsearch/logsearch.properties
   else
     cp /root/test-config/logsearch/logsearch.properties /root/config/logsearch/logsearch.properties
   fi
+
+  if [ "$KNOX" == "true"  ]
+  then
+    cp /root/test-config/logsearch/logsearch-sso.properties /root/config/logsearch/logsearch.properties
+  fi
+
   set_custom_zookeeper_address /root/config/logsearch/logsearch.properties
 }
 
@@ -64,7 +70,7 @@ function create_solr_configs() {
   cp /root/test-config/solr/log4j.properties /root/config/solr/
   cp /root/test-config/solr/zoo.cfg /root/config/solr/
   cp /root/test-config/solr/solr.xml /root/config/solr/
-  if [ $LOGSEARCH_SOLR_SSL_ENABLED == 'true' ]
+  if [ "$LOGSEARCH_SOLR_SSL_ENABLED" == "true" ]
   then
     cp /root/test-config/solr/solr-env-ssl.sh /root/config/solr/solr-env.sh
   else
@@ -79,7 +85,7 @@ function create_configs() {
 }
 
 function generate_keys() {
-  if [ $GENERATE_KEYSTORE_AT_START == 'true' ]
+  if [ "$GENERATE_KEYSTORE_AT_START" == "true" ]
   then
     IP=`hostname --ip-address`
     echo "generating stores for IP: $IP"
@@ -93,7 +99,7 @@ function start_solr_d() {
   /root/solr-$SOLR_VERSION/bin/solr start -cloud -s /root/logsearch_solr_index/data -verbose -force
   touch /var/log/ambari-logsearch-solr/solr.log
 
-  if [ $LOGSEARCH_SOLR_SSL_ENABLED == 'true'  ]
+  if [ "$LOGSEARCH_SOLR_SSL_ENABLED" == "true"  ]
   then
     echo "Setting urlScheme as https and restarting solr..."
     $ZKCLI -zkhost localhost:9983 -cmd clusterprop -name urlScheme -val https
@@ -125,6 +131,26 @@ function start_selenium_server_d() {
   nohup java -jar /root/selenium-server-standalone.jar > /var/log/selenium-test.log &
 }
 
+function start_ldap_d() {
+  if [ "$KNOX" == "true"  ]
+  then
+    echo "KNOX is enabled. Starting Demo LDAP."
+    su knox -c "/ldap.sh"
+  else
+    echo "KNOX is not enabled. Skip Starting Demo LDAP."
+  fi
+}
+
+function start_knox_d() {
+  if [ "$KNOX" == "true"  ]
+  then
+    echo "KNOX is enabled. Starting Demo KNOX gateway."
+    su knox -c "/gateway.sh"
+  else
+    echo "KNOX is not enabled. Skip Starting KNOX gateway."
+  fi
+}
+
 function log() {
   component_log=${COMPONENT_LOG:-"logsearch"}
   case $component_log in
@@ -137,6 +163,12 @@ function log() {
     "selenium")
       tail -f /var/log/selenium-test.log
      ;;
+     "knox")
+      tail -f --retry /knox/logs/gateway.log
+     ;;
+     "ldap")
+      tail -f --retry /knox/logs/ldap.log
+     ;;
      *)
       tail -f /var/log/ambari-logsearch-portal/logsearch-app.log
      ;;
@@ -169,12 +201,28 @@ function main() {
       start_logsearch
       log
      ;;
+     "knox")
+      echo "Start KNOX only ..."
+      export COMPONENT_LOG="knox"
+      export KNOX="true"
+      start_knox_d
+      log
+     ;;
+     "ldap")
+      echo "Start Demo LDAP only ..."
+      export COMPONENT_LOG="ldap"
+      export KNOX="true"
+      start_ldap_d
+      log
+     ;;
      *)
       create_configs
       generate_keys
       start_selenium_server_d
       start_solr_d
       start_logfeeder_d
+      start_ldap_d
+      start_knox_d
       start_logsearch_d
       log
      ;;

ambari-logsearch/docker/knox.yml

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+version: '3.3'
+services:
+  ldap:
+    image: ambari-logsearch:v1.0
+    restart: always
+    hostname: ldap.apache.org
+    networks:
+      - logsearch-network
+    ports:
+      - 33389:33389
+    environment:
+      COMPONENT: ldap
+      COMPONENT_LOG: ldap
+      KNOX: "true"
+  knox:
+    image: ambari-logsearch:v1.0
+    restart: always
+    hostname: knox.apache.org
+    networks:
+      - logsearch-network
+    ports:
+      - 8443:8443
+    volumes:
+      - ./knox/topologies:/knox/conf/topologies
+      - ./knox/logsearch:/knox/data/services/logsearch
+      #- ./knox/applications:/knox/data/applications
+    environment:
+      COMPONENT: knox
+      COMPONENT_LOG: knox
+      KNOX: "true"
+    depends_on:
+      - ldap
+
+networks:
+   logsearch-network:
+      driver: bridge

ambari-logsearch/docker/knox/gateway.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+set -e
+set -o pipefail
+
+nohup java -jar /knox/bin/gateway.jar > /gateway.out &
+