diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLog.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLog.java deleted file mode 100644 index e06653ed4bbe..000000000000 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLog.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.server.http; - -import java.util.HashMap; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogConfigurationException; -import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; -import org.apache.log4j.Appender; -import org.eclipse.jetty.server.AsyncRequestLogWriter; -import org.eclipse.jetty.server.CustomRequestLog; -import org.eclipse.jetty.server.RequestLog; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * RequestLog object for use with Http. - */ -public final class HttpRequestLog { - - private static final Logger LOG = - LoggerFactory.getLogger(HttpRequestLog.class); - private static final HashMap SERVER_TO_COMPONENT; - - private HttpRequestLog() { - } - - static { - SERVER_TO_COMPONENT = new HashMap(); - SERVER_TO_COMPONENT.put("cluster", "resourcemanager"); - SERVER_TO_COMPONENT.put("hdfs", "namenode"); - SERVER_TO_COMPONENT.put("node", "nodemanager"); - } - - public static RequestLog getRequestLog(String name) { - - String lookup = SERVER_TO_COMPONENT.get(name); - if (lookup != null) { - name = lookup; - } - String loggerName = "http.requests." + name; - String appenderName = name + "requestlog"; - Log logger = LogFactory.getLog(loggerName); - - boolean isLog4JLogger; - - try { - isLog4JLogger = logger instanceof Log4JLogger; - } catch (NoClassDefFoundError err) { - // In some dependent projects, log4j may not even be on the classpath at - // runtime, in which case the above instanceof check will throw - // NoClassDefFoundError. - LOG.debug("Could not load Log4JLogger class", err); - isLog4JLogger = false; - } - if (isLog4JLogger) { - Log4JLogger httpLog4JLog = (Log4JLogger) logger; - org.apache.log4j.Logger httpLogger = httpLog4JLog.getLogger(); - Appender appender = null; - - try { - appender = httpLogger.getAppender(appenderName); - } catch (LogConfigurationException e) { - LOG.warn("Http request log for {} could not be created", loggerName); - throw e; - } - - if (appender == null) { - LOG.info("Http request log for {} is not defined", loggerName); - return null; - } - - if (appender instanceof HttpRequestLogAppender) { - HttpRequestLogAppender requestLogAppender - = (HttpRequestLogAppender) appender; - AsyncRequestLogWriter logWriter = new AsyncRequestLogWriter(); - logWriter.setFilename(requestLogAppender.getFilename()); - logWriter.setRetainDays(requestLogAppender.getRetainDays()); - return new CustomRequestLog(logWriter, - CustomRequestLog.EXTENDED_NCSA_FORMAT); - } else { - LOG.warn("Jetty request log for {} was of the wrong class", loggerName); - return null; - } - } else { - LOG.warn("Jetty request log can only be enabled using Log4j"); - return null; - } - } -} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLogAppender.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLogAppender.java deleted file mode 100644 index 9777e2d9b5d2..000000000000 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpRequestLogAppender.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.server.http; - -import org.apache.log4j.AppenderSkeleton; -import org.apache.log4j.spi.LoggingEvent; - -/** - * Log4j Appender adapter for HttpRequestLog. - */ -public class HttpRequestLogAppender extends AppenderSkeleton { - - private String filename; - private int retainDays; - - public HttpRequestLogAppender() { - } - - public void setRetainDays(int retainDays) { - this.retainDays = retainDays; - } - - public int getRetainDays() { - return retainDays; - } - - public void setFilename(String filename) { - this.filename = filename; - } - - public String getFilename() { - return filename; - } - - @Override - public void append(LoggingEvent event) { - } - - @Override - public void close() { - } - - @Override - public boolean requiresLayout() { - return false; - } -} diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java index 241fb5ab81b8..3b76325226b6 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java @@ -89,6 +89,7 @@ import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.CustomRequestLog; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; @@ -96,6 +97,7 @@ import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.Slf4jRequestLogWriter; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.eclipse.jetty.server.handler.HandlerCollection; @@ -618,14 +620,13 @@ private void initializeWebServer(Builder builder) throws IOException { handler.getSessionCookieConfig().setSecure(true); ContextHandlerCollection contexts = new ContextHandlerCollection(); - RequestLog requestLog = HttpRequestLog.getRequestLog(builder.name); - handlers.addHandler(contexts); - if (requestLog != null) { - RequestLogHandler requestLogHandler = new RequestLogHandler(); - requestLogHandler.setRequestLog(requestLog); - handlers.addHandler(requestLogHandler); - } + + RequestLog requestLog = getRequestLog(builder.name); + RequestLogHandler requestLogHandler = new RequestLogHandler(); + requestLogHandler.setRequestLog(requestLog); + handlers.addHandler(requestLogHandler); + handlers.addHandler(webAppContext); final String appDir = getWebAppsPath(builder.name); if (!builder.skipDefaultApps) { @@ -1794,4 +1795,11 @@ public static void setHttpBaseDir(OzoneConfiguration ozoneConfiguration) tmpMetaDir.getAbsolutePath()); } } + + private static RequestLog getRequestLog(String name) { + String loggerName = "http.requests." + name; + Slf4jRequestLogWriter writer = new Slf4jRequestLogWriter(); + writer.setLoggerName(loggerName); + return new CustomRequestLog(writer, CustomRequestLog.EXTENDED_NCSA_FORMAT); + } } diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLog.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLog.java deleted file mode 100644 index eb596acc7a68..000000000000 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLog.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.server.http; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - -import org.apache.log4j.Logger; -import org.eclipse.jetty.server.CustomRequestLog; -import org.eclipse.jetty.server.RequestLog; -import org.junit.jupiter.api.Test; - -/** - * Testing HttpRequestLog. - */ -public class TestHttpRequestLog { - - @Test - public void testAppenderUndefined() { - RequestLog requestLog = HttpRequestLog.getRequestLog("test"); - assertNull(requestLog, "RequestLog should be null"); - } - - @Test - public void testAppenderDefined() { - HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender(); - requestLogAppender.setName("testrequestlog"); - Logger.getLogger("http.requests.test").addAppender(requestLogAppender); - RequestLog requestLog = HttpRequestLog.getRequestLog("test"); - Logger.getLogger("http.requests.test").removeAppender(requestLogAppender); - assertNotNull(requestLog, "RequestLog should not be null"); - assertEquals(CustomRequestLog.class, requestLog.getClass(), - "Class mismatch"); - } -} diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLogAppender.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLogAppender.java deleted file mode 100644 index 418f9215848f..000000000000 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestHttpRequestLogAppender.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdds.server.http; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -/** - * Test Http request log appender. - */ -public class TestHttpRequestLogAppender { - - @Test - public void testParameterPropagation() { - - HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender(); - requestLogAppender.setFilename("jetty-namenode-yyyy_mm_dd.log"); - requestLogAppender.setRetainDays(17); - assertEquals("jetty-namenode-yyyy_mm_dd.log", - requestLogAppender.getFilename(), "Filename mismatch"); - assertEquals(17, requestLogAppender.getRetainDays(), - "Retain days mismatch"); - } -} diff --git a/hadoop-ozone/dist/src/shell/conf/log4j.properties b/hadoop-ozone/dist/src/shell/conf/log4j.properties index aa3d0b4bf43a..ae3c6f51cf3a 100644 --- a/hadoop-ozone/dist/src/shell/conf/log4j.properties +++ b/hadoop-ozone/dist/src/shell/conf/log4j.properties @@ -122,6 +122,25 @@ log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd +# +# HTTP request logs +# +log4j.appender.HttpAccess=org.apache.log4j.DailyRollingFileAppender +log4j.appender.HttpAccess.File=${hadoop.log.dir}/access.log +log4j.appender.HttpAccess.DatePattern=.yyyy-MM-dd +log4j.appender.HttpAccess.layout=org.apache.log4j.PatternLayout +log4j.appender.HttpAccess.layout.ConversionPattern=%m%n + +log4j.additivity.http.requests=false +log4j.logger.http.requests=INFO,HttpAccess +# Create separate appender for each co-hosted component if needed, then enable distinct logger configs: +#log4j.logger.http.requests.hddsDatanode=INFO,HttpAccess +#log4j.logger.http.requests.ozoneManager=INFO,HttpAccess +#log4j.logger.http.requests.recon=INFO,HttpAccess +#log4j.logger.http.requests.s3gateway=INFO,HttpAccess +#log4j.logger.http.requests.s3g-web=INFO,HttpAccess +#log4j.logger.http.requests.scm=INFO,HttpAccess +#log4j.logger.http.requests.webhdfs=INFO,HttpAccess # Custom Logging levels # AWS SDK & S3A FileSystem