[BUILD] Migrate from log4j1 to log4j2

conf/log4j2.properties.template

@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+rootLogger.level = info
+rootLogger.appenderRef.stdout.ref = STDOUT
+
+# Console Appender
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.target = SYSTEM_OUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
+
+appender.console.filter.1.type = Filters
+
+appender.console.filter.1.a.type = ThresholdFilter
+appender.console.filter.1.a.level = info
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+appender.console.filter.1.b.type = RegexFilter
+appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.*
+appender.console.filter.1.b.onMatch = deny
+appender.console.filter.1.b.onMismatch = neutral
+
+# Set the default kyuubi-ctl log level to WARN. When running the kyuubi-ctl, the
+# log level for this class is used to overwrite the root logger's log level.
+logger.ctl.name = org.apache.kyuubi.ctl.ServiceControlCli
+logger.ctl.level = error
+
+# Analysis MySQLFrontend protocol traffic
+# logger.mysql.name = org.apache.kyuubi.server.mysql.codec
+# logger.mysql.level = trace
+
+# Kyuubi BeeLine
+logger.beeline.name = org.apache.hive.beeline.KyuubiBeeLine
+logger.beeline.level = error

dev/dependencyList

@@ -72,6 +72,10 @@ jetty-util/9.4.41.v20210516//jetty-util-9.4.41.v20210516.jar
 jline/0.9.94//jline-0.9.94.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
 libthrift/0.9.3//libthrift-0.9.3.jar
+log4j-1.2-api/2.17.1//log4j-1.2-api-2.17.1.jar
+log4j-api/2.17.1//log4j-api-2.17.1.jar
+log4j-core/2.17.1//log4j-core-2.17.1.jar
+log4j-slf4j-impl/2.17.1//log4j-slf4j-impl-2.17.1.jar
 log4j/1.2.17//log4j-1.2.17.jar
 metrics-core/4.1.1//metrics-core-4.1.1.jar
 metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar

dev/kyuubi-extension-spark-3-1/src/test/resources/log4j2.properties

@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+rootLogger.level = info
+rootLogger.appenderRef.stdout.ref = STDOUT
+rootLogger.appenderRef.file.ref = File
+
+# Console Appender
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.target = SYSTEM_OUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
+
+appender.console.filter.1.type = Filters
+
+appender.console.filter.1.a.type = ThresholdFilter
+appender.console.filter.1.a.level = debug
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+appender.console.filter.1.b.type = RegexFilter
+appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.*
+appender.console.filter.1.b.onMatch = deny
+appender.console.filter.1.b.onMismatch = neutral
+
+# File Appender
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n
+
+appender.file.filter.1.type = Filters
+
+appender.file.filter.1.a.type = RegexFilter
+appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.*
+appender.file.filter.1.a.onMatch = deny
+appender.file.filter.1.a.onMismatch = neutral
+
+# Set the logger level of File Appender to DEBUG
+appender.file.filter.1.b.type = ThresholdFilter
+appender.file.filter.1.b.level = debug

dev/kyuubi-extension-spark-3-2/src/test/resources/log4j2.properties

@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+rootLogger.level = info
+rootLogger.appenderRef.stdout.ref = STDOUT
+rootLogger.appenderRef.file.ref = File
+
+# Console Appender
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.target = SYSTEM_OUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
+
+appender.console.filter.1.type = Filters
+
+appender.console.filter.1.a.type = ThresholdFilter
+appender.console.filter.1.a.level = debug
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+appender.console.filter.1.b.type = RegexFilter
+appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.*
+appender.console.filter.1.b.onMatch = deny
+appender.console.filter.1.b.onMismatch = neutral
+
+# File Appender
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n
+
+appender.file.filter.1.type = Filters
+
+appender.file.filter.1.a.type = RegexFilter
+appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.*
+appender.file.filter.1.a.onMatch = deny
+appender.file.filter.1.a.onMismatch = neutral
+
+# Set the logger level of File Appender to DEBUG
+appender.file.filter.1.b.type = ThresholdFilter
+appender.file.filter.1.b.level = debug

dev/kyuubi-extension-spark-common/src/test/resources/log4j2.properties

@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+rootLogger.level = info
+rootLogger.appenderRef.stdout.ref = STDOUT
+rootLogger.appenderRef.file.ref = File
+
+# Console Appender
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.target = SYSTEM_OUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
+
+appender.console.filter.1.type = Filters
+
+appender.console.filter.1.a.type = ThresholdFilter
+appender.console.filter.1.a.level = debug
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+appender.console.filter.1.b.type = RegexFilter
+appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.*
+appender.console.filter.1.b.onMatch = deny
+appender.console.filter.1.b.onMismatch = neutral
+
+# File Appender
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n
+
+appender.file.filter.1.type = Filters
+
+appender.file.filter.1.a.type = RegexFilter
+appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.*
+appender.file.filter.1.a.onMatch = deny
+appender.file.filter.1.a.onMismatch = neutral
+
+# Set the logger level of File Appender to DEBUG
+appender.file.filter.1.b.type = ThresholdFilter
+appender.file.filter.1.b.level = debug

docs/deployment/settings.md

@@ -363,7 +363,7 @@ Please refer to the Spark official online documentation for [SET Command](http:/
 
 ## Logging
 
-Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging. You can configure it using `$KYUUBI_HOME/conf/log4j.properties`.
+Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging. You can configure it using `$KYUUBI_HOME/conf/log4j2.properties`.
 ```bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -382,22 +382,40 @@ Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging. You can
 # limitations under the License.
 #
 
-# Set everything to be logged to the console
-log4j.rootCategory=INFO, console
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c{2}: %m%n
+# Set everything to be logged to the file target/unit-tests.log
+rootLogger.level = info
+rootLogger.appenderRef.stdout.ref = STDOUT
+
+# Console Appender
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.target = SYSTEM_OUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
+
+appender.console.filter.1.type = Filters
+
+appender.console.filter.1.a.type = ThresholdFilter
+appender.console.filter.1.a.level = info
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+appender.console.filter.1.b.type = RegexFilter
+appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.*
+appender.console.filter.1.b.onMatch = deny
+appender.console.filter.1.b.onMismatch = neutral
 
 # Set the default kyuubi-ctl log level to WARN. When running the kyuubi-ctl, the
 # log level for this class is used to overwrite the root logger's log level.
-log4j.logger.org.apache.kyuubi.ctl.ServiceControlCli=ERROR
+logger.ctl.name = org.apache.kyuubi.ctl.ServiceControlCli
+logger.ctl.level = error
 
 # Analysis MySQLFrontend protocol traffic
-# log4j.logger.org.apache.kyuubi.server.mysql.codec=TRACE
+# logger.mysql.name = org.apache.kyuubi.server.mysql.codec
+# logger.mysql.level = trace
 
 # Kyuubi BeeLine
-log4j.logger.org.apache.hive.beeline.KyuubiBeeLine=ERROR
+logger.beeline.name = org.apache.hive.beeline.KyuubiBeeLine
+logger.beeline.level = error
 ```
 
 ## Other Configurations

externals/kyuubi-flink-sql-engine/src/test/resources/log4j2.properties

@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the file target/unit-tests.log
+rootLogger.level = debug
+rootLogger.appenderRef.stdout.ref = STDOUT
+rootLogger.appenderRef.file.ref = File
+
+# Console Appender
+appender.console.type = Console
+appender.console.name = STDOUT
+appender.console.target = SYSTEM_OUT
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
+
+appender.console.filter.1.type = Filters
+
+appender.console.filter.1.a.type = ThresholdFilter
+appender.console.filter.1.a.level = debug
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+appender.console.filter.1.b.type = RegexFilter
+appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.*
+appender.console.filter.1.b.onMatch = deny
+appender.console.filter.1.b.onMismatch = neutral
+
+# File Appender
+appender.file.type = File
+appender.file.name = File
+appender.file.fileName = target/unit-tests.log
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n
+
+appender.file.filter.1.type = Filters
+
+appender.file.filter.1.a.type = RegexFilter
+appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.*
+appender.file.filter.1.a.onMatch = deny
+appender.file.filter.1.a.onMismatch = neutral
+
+# Set the logger level of File Appender to DEBUG
+appender.file.filter.1.b.type = ThresholdFilter
+appender.file.filter.1.b.level = debug