diff --git a/conf/log4j2.properties.template b/conf/log4j2.properties.template new file mode 100644 index 00000000000..04d7bdedcd2 --- /dev/null +++ b/conf/log4j2.properties.template @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = info + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# Set the default kyuubi-ctl log level to WARN. When running the kyuubi-ctl, the +# log level for this class is used to overwrite the root logger's log level. +logger.ctl.name = org.apache.kyuubi.ctl.ServiceControlCli +logger.ctl.level = error + +# Analysis MySQLFrontend protocol traffic +# logger.mysql.name = org.apache.kyuubi.server.mysql.codec +# logger.mysql.level = trace + +# Kyuubi BeeLine +logger.beeline.name = org.apache.hive.beeline.KyuubiBeeLine +logger.beeline.level = error diff --git a/dev/dependencyList b/dev/dependencyList index 7f3b9426b63..8ef7fb7bfa1 100644 --- a/dev/dependencyList +++ b/dev/dependencyList @@ -72,6 +72,10 @@ jetty-util/9.4.41.v20210516//jetty-util-9.4.41.v20210516.jar jline/0.9.94//jline-0.9.94.jar libfb303/0.9.3//libfb303-0.9.3.jar libthrift/0.9.3//libthrift-0.9.3.jar +log4j-1.2-api/2.17.1//log4j-1.2-api-2.17.1.jar +log4j-api/2.17.1//log4j-api-2.17.1.jar +log4j-core/2.17.1//log4j-core-2.17.1.jar +log4j-slf4j-impl/2.17.1//log4j-slf4j-impl-2.17.1.jar log4j/1.2.17//log4j-1.2.17.jar metrics-core/4.1.1//metrics-core-4.1.1.jar metrics-jmx/4.1.1//metrics-jmx-4.1.1.jar diff --git a/dev/kyuubi-extension-spark-3-1/src/test/resources/log4j2.properties b/dev/kyuubi-extension-spark-3-1/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..6f4ea28ab84 --- /dev/null +++ b/dev/kyuubi-extension-spark-3-1/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/dev/kyuubi-extension-spark-3-2/src/test/resources/log4j2.properties b/dev/kyuubi-extension-spark-3-2/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..6f4ea28ab84 --- /dev/null +++ b/dev/kyuubi-extension-spark-3-2/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/dev/kyuubi-extension-spark-common/src/test/resources/log4j2.properties b/dev/kyuubi-extension-spark-common/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..6f4ea28ab84 --- /dev/null +++ b/dev/kyuubi-extension-spark-common/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md index 27b07cd0da7..60ac64ebaa1 100644 --- a/docs/deployment/settings.md +++ b/docs/deployment/settings.md @@ -363,7 +363,7 @@ Please refer to the Spark official online documentation for [SET Command](http:/ ## Logging -Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging. You can configure it using `$KYUUBI_HOME/conf/log4j.properties`. +Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging. You can configure it using `$KYUUBI_HOME/conf/log4j2.properties`. ```bash # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -382,22 +382,40 @@ Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging. You can # limitations under the License. # -# Set everything to be logged to the console -log4j.rootCategory=INFO, console -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %p %c{2}: %m%n +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = info + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral # Set the default kyuubi-ctl log level to WARN. When running the kyuubi-ctl, the # log level for this class is used to overwrite the root logger's log level. -log4j.logger.org.apache.kyuubi.ctl.ServiceControlCli=ERROR +logger.ctl.name = org.apache.kyuubi.ctl.ServiceControlCli +logger.ctl.level = error # Analysis MySQLFrontend protocol traffic -# log4j.logger.org.apache.kyuubi.server.mysql.codec=TRACE +# logger.mysql.name = org.apache.kyuubi.server.mysql.codec +# logger.mysql.level = trace # Kyuubi BeeLine -log4j.logger.org.apache.hive.beeline.KyuubiBeeLine=ERROR +logger.beeline.name = org.apache.hive.beeline.KyuubiBeeLine +logger.beeline.level = error ``` ## Other Configurations diff --git a/externals/kyuubi-flink-sql-engine/src/test/resources/log4j2.properties b/externals/kyuubi-flink-sql-engine/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..fafe9b2a1ad --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/externals/kyuubi-spark-sql-engine/src/test/resources/log4j2.properties b/externals/kyuubi-spark-sql-engine/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..fafe9b2a1ad --- /dev/null +++ b/externals/kyuubi-spark-sql-engine/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/externals/kyuubi-trino-engine/src/test/resources/log4j2.properties b/externals/kyuubi-trino-engine/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..fafe9b2a1ad --- /dev/null +++ b/externals/kyuubi-trino-engine/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/integration-tests/kyuubi-kubernetes-deployment-it/src/test/resources/log4j2.properties b/integration-tests/kyuubi-kubernetes-deployment-it/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..45d900984b0 --- /dev/null +++ b/integration-tests/kyuubi-kubernetes-deployment-it/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to INFO +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = info diff --git a/kyuubi-assembly/pom.xml b/kyuubi-assembly/pom.xml index baba1db1a5b..f7150cf8306 100644 --- a/kyuubi-assembly/pom.xml +++ b/kyuubi-assembly/pom.xml @@ -107,8 +107,23 @@ - org.slf4j - slf4j-log4j12 + org.apache.logging.log4j + log4j-slf4j-impl + + + + org.apache.logging.log4j + log4j-api + + + + org.apache.logging.log4j + log4j-core + + + + org.apache.logging.log4j + log4j-1.2-api diff --git a/kyuubi-common/pom.xml b/kyuubi-common/pom.xml index a850cad7759..a4fbadb3c14 100644 --- a/kyuubi-common/pom.xml +++ b/kyuubi-common/pom.xml @@ -48,8 +48,23 @@ - org.slf4j - slf4j-log4j12 + org.apache.logging.log4j + log4j-slf4j-impl + + + + org.apache.logging.log4j + log4j-api + + + + org.apache.logging.log4j + log4j-core + + + + org.apache.logging.log4j + log4j-1.2-api diff --git a/kyuubi-common/src/main/resources/log4j2-defaults.properties b/kyuubi-common/src/main/resources/log4j2-defaults.properties new file mode 100644 index 00000000000..ab54692b486 --- /dev/null +++ b/kyuubi-common/src/main/resources/log4j2-defaults.properties @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the console +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala index 4ec783dfaeb..9bf9045788e 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala @@ -17,7 +17,8 @@ package org.apache.kyuubi -import org.apache.log4j.{Level, LogManager, PropertyConfigurator} +import org.apache.logging.log4j.{Level, LogManager} +import org.apache.logging.log4j.core.LoggerContext import org.slf4j.{Logger, LoggerFactory} import org.slf4j.impl.StaticLoggerBinder @@ -90,32 +91,64 @@ trait Logging { } private def initializeLogging(isInterpreter: Boolean): Unit = { - if (Logging.isLog4j12) { - val log4j12Initialized = LogManager.getRootLogger.getAllAppenders.hasMoreElements + if (Logging.isLog4j2) { + val log4j2Initialized = !LogManager.getRootLogger + .asInstanceOf[org.apache.logging.log4j.core.Logger].getAppenders.isEmpty // scalastyle:off println - if (!log4j12Initialized) { + if (!log4j2Initialized) { Logging.useDefault = true - val defaultLogProps = "log4j-defaults.properties" + val defaultLogProps = "log4j2-defaults.properties" Option(Thread.currentThread().getContextClassLoader.getResource(defaultLogProps)) match { case Some(url) => - PropertyConfigurator.configure(url) + val context = LogManager.getContext(false).asInstanceOf[LoggerContext] + context.setConfigLocation(url.toURI) case None => System.err.println(s"Missing $defaultLogProps") } } val rootLogger = LogManager.getRootLogger + .asInstanceOf[org.apache.logging.log4j.core.Logger] if (Logging.defaultRootLevel == null) { - Logging.defaultRootLevel = rootLogger.getLevel + Logging.defaultRootLevel = rootLogger.getLevel.toString } if (isInterpreter) { // set kyuubi ctl log level, default ERROR val ctlLogger = LogManager.getLogger(loggerName) + .asInstanceOf[org.apache.logging.log4j.core.Logger] val ctlLevel = Option(ctlLogger.getLevel()).getOrElse(Level.ERROR) rootLogger.setLevel(ctlLevel) } // scalastyle:on println + } else if (Logging.isLog4j12) { + val log4j12Initialized = + org.apache.log4j.LogManager.getRootLogger.getAllAppenders.hasMoreElements + // scalastyle:off println + if (!log4j12Initialized) { + Logging.useDefault = true + val defaultLogProps = "log4j-defaults.properties" + Option(Thread.currentThread().getContextClassLoader.getResource(defaultLogProps)) match { + case Some(url) => + org.apache.log4j.PropertyConfigurator.configure(url) + + case None => + System.err.println(s"Missing $defaultLogProps") + } + + val rootLogger = org.apache.log4j.LogManager.getRootLogger + if (Logging.defaultRootLevel == null) { + Logging.defaultRootLevel = rootLogger.getLevel.toString + } + + if (isInterpreter) { + // set kyuubi ctl log level, default ERROR + val ctlLogger = org.apache.log4j.LogManager.getLogger(loggerName) + val ctlLevel = Option(ctlLogger.getLevel()).getOrElse(org.apache.log4j.Level.ERROR) + rootLogger.setLevel(ctlLevel) + } + // scalastyle:on println + } } Logging.initialized = true @@ -127,14 +160,23 @@ trait Logging { object Logging { @volatile private var useDefault = false - @volatile private var defaultRootLevel: Level = _ + @volatile private var defaultRootLevel: String = _ @volatile private var initialized = false val initLock = new Object() - private def isLog4j12: Boolean = { + + private[kyuubi] def isLog4j12: Boolean = { // This distinguishes the log4j 1.2 binding, currently // org.slf4j.impl.Log4jLoggerFactory, from the log4j 2.0 binding, currently // org.apache.logging.slf4j.Log4jLoggerFactory val binderClass = StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr "org.slf4j.impl.Log4jLoggerFactory".equals(binderClass) } + + private[kyuubi] def isLog4j2: Boolean = { + // This distinguishes the log4j 1.2 binding, currently + // org.slf4j.impl.Log4jLoggerFactory, from the log4j 2.0 binding, currently + // org.apache.logging.slf4j.Log4jLoggerFactory + val binderClass = StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr + "org.apache.logging.slf4j.Log4jLoggerFactory".equals(binderClass) + } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j12DivertAppender.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j12DivertAppender.scala new file mode 100644 index 00000000000..1191e94ae29 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j12DivertAppender.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.operation.log + +import java.io.CharArrayWriter + +import scala.collection.JavaConverters._ + +import org.apache.log4j._ +import org.apache.log4j.spi.{Filter, LoggingEvent} + +class Log4j12DivertAppender extends WriterAppender { + + final private val writer = new CharArrayWriter + + final private val lo = Logger.getRootLogger + .getAllAppenders.asScala + .find(_.isInstanceOf[ConsoleAppender]) + .map(_.asInstanceOf[Appender].getLayout) + .getOrElse(new PatternLayout("%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n")) + + setName("KyuubiSparkSQLEngineLogDivertAppender") + setWriter(writer) + setLayout(lo) + + addFilter { _: LoggingEvent => + if (OperationLog.getCurrentOperationLog == null) Filter.DENY else Filter.NEUTRAL + } + + /** + * Overrides WriterAppender.subAppend(), which does the real logging. No need + * to worry about concurrency since log4j calls this synchronously. + */ + override protected def subAppend(event: LoggingEvent): Unit = { + super.subAppend(event) + // That should've gone into our writer. Notify the LogContext. + val logOutput = writer.toString + writer.reset() + val log = OperationLog.getCurrentOperationLog + if (log != null) log.write(logOutput) + } +} + +object Log4j12DivertAppender { + def initialize(): Unit = { + org.apache.log4j.Logger.getRootLogger.addAppender(new Log4j12DivertAppender()) + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala new file mode 100644 index 00000000000..82ce6feb962 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.operation.log + +import java.io.CharArrayWriter + +import scala.collection.JavaConverters._ + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.core.{Filter, LogEvent, StringLayout} +import org.apache.logging.log4j.core.appender.{AbstractWriterAppender, ConsoleAppender, WriterManager} +import org.apache.logging.log4j.core.config.Property +import org.apache.logging.log4j.core.filter.AbstractFilter +import org.apache.logging.log4j.core.layout.PatternLayout + +class Log4j2DivertAppender( + name: String, + layout: StringLayout, + filter: Filter, + ignoreExceptions: Boolean, + immediateFlush: Boolean, + writer: CharArrayWriter) + extends AbstractWriterAppender[WriterManager]( + name, + layout, + filter, + ignoreExceptions, + immediateFlush, + Property.EMPTY_ARRAY, + new WriterManager(writer, name, layout, true)) { + def this() = this( + "KyuubiEngineLogDivertAppender", + Log4j2DivertAppender.initLayout(), + null, + false, + true, + new CharArrayWriter()) + + addFilter(new AbstractFilter() { + override def filter(event: LogEvent): Filter.Result = { + if (OperationLog.getCurrentOperationLog == null) { + Filter.Result.DENY + } else { + Filter.Result.NEUTRAL + } + } + }) + + def initLayout(): StringLayout = { + LogManager.getRootLogger.asInstanceOf[org.apache.logging.log4j.core.Logger] + .getAppenders.values().asScala + .find(ap => ap.isInstanceOf[ConsoleAppender] && ap.getLayout.isInstanceOf[StringLayout]) + .map(_.getLayout.asInstanceOf[StringLayout]) + .getOrElse(PatternLayout.newBuilder().withPattern( + "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n").build()) + } + + /** + * Overrides AbstractWriterAppender.append(), which does the real logging. No need + * to worry about concurrency since log4j calls this synchronously. + */ + override def append(event: LogEvent): Unit = { + super.append(event) + // That should've gone into our writer. Notify the LogContext. + val logOutput = writer.toString + writer.reset() + val log = OperationLog.getCurrentOperationLog + if (log != null) log.write(logOutput) + } +} + +object Log4j2DivertAppender { + def initLayout(): StringLayout = { + LogManager.getRootLogger.asInstanceOf[org.apache.logging.log4j.core.Logger] + .getAppenders.values().asScala + .find(ap => ap.isInstanceOf[ConsoleAppender] && ap.getLayout.isInstanceOf[StringLayout]) + .map(_.getLayout.asInstanceOf[StringLayout]) + .getOrElse(PatternLayout.newBuilder().withPattern( + "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n").build()) + } + + def initialize(): Unit = { + val ap = new Log4j2DivertAppender() + org.apache.logging.log4j.LogManager.getRootLogger() + .asInstanceOf[org.apache.logging.log4j.core.Logger].addAppender(ap) + ap.start() + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/LogDivertAppender.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/LogDivertAppender.scala index 57585df88d9..6766962fe93 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/LogDivertAppender.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/LogDivertAppender.scala @@ -17,47 +17,18 @@ package org.apache.kyuubi.operation.log -import java.io.CharArrayWriter +import org.slf4j.impl.StaticLoggerBinder -import scala.collection.JavaConverters._ +import org.apache.kyuubi.Logging -import org.apache.log4j._ -import org.apache.log4j.spi.{Filter, LoggingEvent} - -class LogDivertAppender extends WriterAppender { - - final private val writer = new CharArrayWriter - - final private val lo = Logger.getRootLogger - .getAllAppenders.asScala - .find(_.isInstanceOf[ConsoleAppender]) - .map(_.asInstanceOf[Appender].getLayout) - .getOrElse(new PatternLayout("%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n")) - - setName("KyuubiSparkSQLEngineLogDivertAppender") - setWriter(writer) - setLayout(lo) - - addFilter { _: LoggingEvent => - if (OperationLog.getCurrentOperationLog == null) Filter.DENY else Filter.NEUTRAL - } - - /** - * Overrides WriterAppender.subAppend(), which does the real logging. No need - * to worry about concurrency since log4j calls this synchronously. - */ - override protected def subAppend(event: LoggingEvent): Unit = { - super.subAppend(event) - // That should've gone into our writer. Notify the LogContext. - val logOutput = writer.toString - writer.reset() - val log = OperationLog.getCurrentOperationLog - if (log != null) log.write(logOutput) - } -} - -object LogDivertAppender { +object LogDivertAppender extends Logging { def initialize(): Unit = { - org.apache.log4j.Logger.getRootLogger.addAppender(new LogDivertAppender()) + if (Logging.isLog4j2) { + Log4j2DivertAppender.initialize() + } else if (Logging.isLog4j12) { + Log4j12DivertAppender.initialize() + } else { + warn(s"Unsupported SLF4J binding ${StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr}") + } } } diff --git a/kyuubi-common/src/test/resources/log4j.properties b/kyuubi-common/src/test/resources/log4j.properties index dc7155f258e..fd7d4e8a873 100644 --- a/kyuubi-common/src/test/resources/log4j.properties +++ b/kyuubi-common/src/test/resources/log4j.properties @@ -31,7 +31,7 @@ log4j.appender.FA.file=target/unit-tests.log log4j.appender.FA.layout=org.apache.log4j.PatternLayout log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n -# Set the logger level of File Appender to WARN +# Set the logger level of File Appender to DEBUG log4j.appender.FA.Threshold = DEBUG # SPARK-34128: Suppress undesirable TTransportException warnings involved in THRIFT-4805 diff --git a/kyuubi-common/src/test/resources/log4j2.properties b/kyuubi-common/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..fafe9b2a1ad --- /dev/null +++ b/kyuubi-common/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala index 02927830523..d4b5fdf59d6 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala @@ -20,8 +20,10 @@ package org.apache.kyuubi import scala.collection.mutable.ArrayBuffer // scalastyle:off -import org.apache.log4j.{Appender, AppenderSkeleton, Level, Logger} -import org.apache.log4j.spi.LoggingEvent +import org.apache.logging.log4j._ +import org.apache.logging.log4j.core.{LogEvent, Logger, LoggerContext} +import org.apache.logging.log4j.core.appender.AbstractAppender +import org.apache.logging.log4j.core.config.Property import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, Outcome} import org.scalatest.concurrent.Eventually import org.scalatest.funsuite.AnyFunSuite @@ -70,38 +72,66 @@ trait KyuubiFunSuite extends AnyFunSuite * appender and restores the log level if necessary. */ final def withLogAppender( - appender: Appender, - loggerName: Option[String] = None, + appender: AbstractAppender, + loggerNames: Seq[String] = Seq.empty, level: Option[Level] = None)( f: => Unit): Unit = { - val logger = loggerName.map(Logger.getLogger).getOrElse(Logger.getRootLogger) - val restoreLevel = logger.getLevel - logger.addAppender(appender) - if (level.isDefined) { - logger.setLevel(level.get) + val loggers = + if (loggerNames.nonEmpty) { + loggerNames.map(LogManager.getLogger) + } else { + Seq(LogManager.getRootLogger) + } + val restoreLevels = loggers.map(_.getLevel) + loggers.foreach { l => + val logger = l.asInstanceOf[Logger] + logger.addAppender(appender) + appender.start() + if (level.isDefined) { + logger.setLevel(level.get) + logger.get().setLevel(level.get) + LogManager.getContext(false).asInstanceOf[LoggerContext].updateLoggers() + } } try f finally { - logger.removeAppender(appender) + loggers.foreach(_.asInstanceOf[Logger].removeAppender(appender)) + appender.stop() if (level.isDefined) { - logger.setLevel(restoreLevel) + loggers.zipWithIndex.foreach { case (logger, i) => + logger.asInstanceOf[Logger].setLevel(restoreLevels(i)) + logger.asInstanceOf[Logger].get().setLevel(restoreLevels(i)) + } } } } - class LogAppender(msg: String = "", maxEvents: Int = 1000) extends AppenderSkeleton { - val loggingEvents = new ArrayBuffer[LoggingEvent]() + class LogAppender(msg: String = "", maxEvents: Int = 1000) + extends AbstractAppender("logAppender", null, null, true, Property.EMPTY_ARRAY) { + private val _loggingEvents = new ArrayBuffer[LogEvent]() + private var _threshold: Level = Level.INFO - override def append(loggingEvent: LoggingEvent): Unit = { - if (loggingEvents.size >= maxEvents) { - val loggingInfo = if (msg == "") "." else s" while logging $msg." - throw new IllegalStateException( - s"Number of events reached the limit of $maxEvents$loggingInfo") + override def append(loggingEvent: LogEvent): Unit = loggingEvent.synchronized { + val copyEvent = loggingEvent.toImmutable + if (copyEvent.getLevel.isMoreSpecificThan(_threshold)) { + _loggingEvents.synchronized { + if (_loggingEvents.size >= maxEvents) { + val loggingInfo = if (msg == "") "." else s" while logging $msg." + throw new IllegalStateException( + s"Number of events reached the limit of $maxEvents$loggingInfo") + } + _loggingEvents.append(copyEvent) + } } - loggingEvents.append(loggingEvent) } - override def close(): Unit = {} - override def requiresLayout(): Boolean = false + + def setThreshold(threshold: Level): Unit = { + _threshold = threshold + } + + def loggingEvents: ArrayBuffer[LogEvent] = _loggingEvents.synchronized { + _loggingEvents.filterNot(_ == null) + } } final def withSystemProperty(key: String, value: String)(f: => Unit): Unit = { diff --git a/kyuubi-ctl/src/test/resources/log4j.properties b/kyuubi-ctl/src/test/resources/log4j.properties index dc7155f258e..fd7d4e8a873 100644 --- a/kyuubi-ctl/src/test/resources/log4j.properties +++ b/kyuubi-ctl/src/test/resources/log4j.properties @@ -31,7 +31,7 @@ log4j.appender.FA.file=target/unit-tests.log log4j.appender.FA.layout=org.apache.log4j.PatternLayout log4j.appender.FA.layout.ConversionPattern=%d{HH:mm:ss.SSS} %t %p %c{2}: %m%n -# Set the logger level of File Appender to WARN +# Set the logger level of File Appender to DEBUG log4j.appender.FA.Threshold = DEBUG # SPARK-34128: Suppress undesirable TTransportException warnings involved in THRIFT-4805 diff --git a/kyuubi-ctl/src/test/resources/log4j2.properties b/kyuubi-ctl/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..fafe9b2a1ad --- /dev/null +++ b/kyuubi-ctl/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ServiceControlCliArgumentsSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ServiceControlCliArgumentsSuite.scala index ec5d36a1dbd..fb3dfb416b7 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ServiceControlCliArgumentsSuite.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ServiceControlCliArgumentsSuite.scala @@ -42,7 +42,8 @@ class ServiceControlCliArgumentsSuite extends KyuubiFunSuite { } thread.start() thread.join() - assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(searchString))) + assert(logAppender.loggingEvents.exists( + _.getMessage.getFormattedMessage.contains(searchString))) } } @@ -66,7 +67,8 @@ class ServiceControlCliArgumentsSuite extends KyuubiFunSuite { } thread.start() thread.join() - assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(searchString))) + assert(logAppender.loggingEvents.exists( + _.getMessage.getFormattedMessage.contains(searchString))) } } diff --git a/kyuubi-ha/src/test/resources/log4j2.properties b/kyuubi-ha/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..fafe9b2a1ad --- /dev/null +++ b/kyuubi-ha/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ServiceDiscoverySuite.scala b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ServiceDiscoverySuite.scala index c1240fd6fef..104d1147a39 100644 --- a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ServiceDiscoverySuite.scala +++ b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/ServiceDiscoverySuite.scala @@ -191,7 +191,8 @@ class ServiceDiscoverySuite extends KerberizedTestHelper { assert(server.getServiceState === ServiceState.STOPPED) val msg = s"This Kyuubi instance ${server.frontendServices.head.connectionUrl}" + s" is now de-registered" - assert(logAppender.loggingEvents.exists(_.getRenderedMessage.contains(msg))) + assert(logAppender.loggingEvents.exists( + _.getMessage.getFormattedMessage.contains(msg))) } } finally { server.stop() diff --git a/kyuubi-hive-beeline/pom.xml b/kyuubi-hive-beeline/pom.xml index 95e394f9b04..1af51688e39 100644 --- a/kyuubi-hive-beeline/pom.xml +++ b/kyuubi-hive-beeline/pom.xml @@ -102,8 +102,23 @@ - org.slf4j - slf4j-log4j12 + org.apache.logging.log4j + log4j-slf4j-impl + + + + org.apache.logging.log4j + log4j-api + + + + org.apache.logging.log4j + log4j-core + + + + org.apache.logging.log4j + log4j-1.2-api diff --git a/kyuubi-metrics/src/test/resources/log4j2.properties b/kyuubi-metrics/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..fafe9b2a1ad --- /dev/null +++ b/kyuubi-metrics/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/kyuubi-server/src/test/resources/log4j2.properties b/kyuubi-server/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..45d900984b0 --- /dev/null +++ b/kyuubi-server/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = info +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to INFO +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = info diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala index 1f8359e44a6..3972554c216 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/config/AllKyuubiConfiguration.scala @@ -222,9 +222,9 @@ class AllKyuubiConfiguration extends KyuubiFunSuite { newOutput += ("## Logging") newOutput += "" newOutput += ("Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging." + - " You can configure it using `$KYUUBI_HOME/conf/log4j.properties`.") + " You can configure it using `$KYUUBI_HOME/conf/log4j2.properties`.") - rewriteToConf(Paths.get(kyuubiHome, "conf", "log4j.properties.template"), newOutput) + rewriteToConf(Paths.get(kyuubiHome, "conf", "log4j2.properties.template"), newOutput) newOutput += "" newOutput += ("## Other Configurations") diff --git a/kyuubi-zookeeper/src/test/resources/log4j2.properties b/kyuubi-zookeeper/src/test/resources/log4j2.properties new file mode 100644 index 00000000000..fafe9b2a1ad --- /dev/null +++ b/kyuubi-zookeeper/src/test/resources/log4j2.properties @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/unit-tests.log +rootLogger.level = debug +rootLogger.appenderRef.stdout.ref = STDOUT +rootLogger.appenderRef.file.ref = File + +# Console Appender +appender.console.type = Console +appender.console.name = STDOUT +appender.console.target = SYSTEM_OUT +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n + +appender.console.filter.1.type = Filters + +appender.console.filter.1.a.type = ThresholdFilter +appender.console.filter.1.a.level = debug + +# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805 +appender.console.filter.1.b.type = RegexFilter +appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.* +appender.console.filter.1.b.onMatch = deny +appender.console.filter.1.b.onMismatch = neutral + +# File Appender +appender.file.type = File +appender.file.name = File +appender.file.fileName = target/unit-tests.log +appender.file.layout.type = PatternLayout +appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n + +appender.file.filter.1.type = Filters + +appender.file.filter.1.a.type = RegexFilter +appender.file.filter.1.a.regx = .*Thrift error occurred during processing of message.* +appender.file.filter.1.a.onMatch = deny +appender.file.filter.1.a.onMismatch = neutral + +# Set the logger level of File Appender to DEBUG +appender.file.filter.1.b.type = ThresholdFilter +appender.file.filter.1.b.level = debug diff --git a/pom.xml b/pom.xml index 4a2360baab4..e7b11d3b8a7 100644 --- a/pom.xml +++ b/pom.xml @@ -131,6 +131,7 @@ 3.2.9 4.0.1 1.7.30 + 2.17.1 3.1.2 spark-${spark.version}-bin-hadoop${hadoop.binary.version}.tgz https://archive.apache.org/dist/spark/spark-${spark.version} @@ -365,6 +366,14 @@ org.apache.hadoop hadoop-client + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + @@ -567,14 +576,30 @@ org.slf4j - slf4j-log4j12 + jul-to-slf4j ${slf4j.version} - org.slf4j - jul-to-slf4j - ${slf4j.version} + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version}