diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 3aa9bdbcc66a2..6ad5a83514781 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -9,6 +9,8 @@ on: branches: - master - 'release-*' +env: + MVN_ARGS: -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn jobs: build: @@ -54,14 +56,14 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} FLINK_PROFILE: ${{ matrix.flinkProfile }} run: - mvn clean install -Pintegration-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DskipTests=true -B -V + mvn clean install -Pintegration-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DskipTests=true $MVN_ARGS - name: Quickstart Test env: SCALA_PROFILE: ${{ matrix.scalaProfile }} SPARK_PROFILE: ${{ matrix.sparkProfile }} FLINK_PROFILE: ${{ matrix.flinkProfile }} run: - mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DfailIfNoTests=false -pl hudi-examples/hudi-examples-flink,hudi-examples/hudi-examples-java,hudi-examples/hudi-examples-spark + mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DfailIfNoTests=false -pl hudi-examples/hudi-examples-flink,hudi-examples/hudi-examples-java,hudi-examples/hudi-examples-spark $MVN_ARGS - name: Spark SQL Test env: SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -69,4 +71,4 @@ jobs: FLINK_PROFILE: ${{ matrix.flinkProfile }} if: ${{ !endsWith(env.SPARK_PROFILE, '2.4') }} # skip test spark 2.4 as it's covered by Azure CI run: - mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" '-Dtest=Test*' -pl hudi-spark-datasource/hudi-spark + mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" '-Dtest=Test*' -pl hudi-spark-datasource/hudi-spark $MVN_ARGS diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7cf83fed50e22..75dc6b48b5121 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -73,9 +73,9 @@ parameters: variables: BUILD_PROFILES: '-Dscala-2.11 -Dspark2 -Dflink1.14' - PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true' + PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn' MVN_OPTS_INSTALL: '-DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS)' - MVN_OPTS_TEST: '-fae $(BUILD_PROFILES) $(PLUGIN_OPTS)' + MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)' SPARK_VERSION: '2.4.4' HADOOP_VERSION: '2.7' SPARK_ARCHIVE: spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION) diff --git a/docker/demo/config/log4j.properties b/docker/demo/config/log4j.properties deleted file mode 100644 index df8ad3d15e07e..0000000000000 --- a/docker/demo/config/log4j.properties +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Set everything to be logged to the console -log4j.rootCategory=WARN, console -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n -# Set the default spark-shell log level to WARN. When running the spark-shell, the -# log level for this class is used to overwrite the root logger's log level, so that -# the user can have different defaults for the shell and regular Spark apps. -log4j.logger.org.apache.spark.repl.Main=WARN -# Set logging of integration testsuite to INFO level -log4j.logger.org.apache.hudi.integ.testsuite=INFO -# Settings to quiet third party logs that are too verbose -log4j.logger.org.spark_project.jetty=WARN -log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR -log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO -log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO -log4j.logger.org.apache.parquet=ERROR -log4j.logger.parquet=ERROR -log4j.logger.org.apache.spark=WARN -# Disabling Jetty logs -log4j.logger.org.apache.hudi.org.eclipse.jetty=ERROR -# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support -log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL -log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR \ No newline at end of file diff --git a/docker/demo/config/log4j2.properties b/docker/demo/config/log4j2.properties new file mode 100644 index 0000000000000..dd3c1ff9ef7a2 --- /dev/null +++ b/docker/demo/config/log4j2.properties @@ -0,0 +1,60 @@ +### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +### +status = warn +name = HudiConsoleLog + +# Set everything to be logged to the console +appender.console.type = Console +appender.console.name = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + +# Root logger level +rootLogger.level = warn +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = CONSOLE + +# Set the default spark-shell log level to WARN. When running the spark-shell, the +# log level for this class is used to overwrite the root logger's log level, so that +# the user can have different defaults for the shell and regular Spark apps. +logger.apache_spark_repl.name = org.apache.spark.repl.Main +logger.apache_spark_repl.level = warn +# Set logging of integration testsuite to INFO level +logger.hudi_integ.name = org.apache.hudi.integ.testsuite +logger.hudi_integ.level = info +# Settings to quiet third party logs that are too verbose +logger.apache_spark_jetty.name = org.spark_project.jetty +logger.apache_spark_jetty.level = warn +logger.apache_spark_jett_lifecycle.name = org.spark_project.jetty.util.component.AbstractLifeCycle +logger.apache_spark_jett_lifecycle.level = error +logger.apache_spark_repl_imain.name = org.apache.spark.repl.SparkIMain$exprTyper +logger.apache_spark_repl_imain.level = info +logger.apache_spark_repl_iloop.name = org.apache.spark.repl.SparkILoop$SparkILoopInterpreter +logger.apache_spark_repl_iloop.level = info +logger.parquet.name = org.apache.parquet +logger.parquet.level = error +logger.spark.name = org.apache.spark +logger.spark.level = warn +# Disabling Jetty logs +logger.jetty.name = org.apache.hudi.org.eclipse.jetty +logger.jetty.level = error +# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support +logger.hive_handler.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler +logger.hive_handler.level = fatal +logger.hive_func_registry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry +logger.hive_func_registry.level = error \ No newline at end of file diff --git a/docker/demo/setup_demo_container.sh b/docker/demo/setup_demo_container.sh index e3ba231771b08..ecec2e99f713c 100755 --- a/docker/demo/setup_demo_container.sh +++ b/docker/demo/setup_demo_container.sh @@ -17,7 +17,7 @@ echo "Copying spark default config and setting up configs" cp /var/hoodie/ws/docker/demo/config/spark-defaults.conf $SPARK_CONF_DIR/. -cp /var/hoodie/ws/docker/demo/config/log4j.properties $SPARK_CONF_DIR/. +cp /var/hoodie/ws/docker/demo/config/log4j2.properties $SPARK_CONF_DIR/. hadoop fs -mkdir -p /var/demo/ hadoop fs -mkdir -p /tmp/spark-events hadoop fs -copyFromLocal -f /var/hoodie/ws/docker/demo/config /var/demo/. diff --git a/hudi-aws/pom.xml b/hudi-aws/pom.xml index 0152583bad757..eac2c60d30066 100644 --- a/hudi-aws/pom.xml +++ b/hudi-aws/pom.xml @@ -46,20 +46,6 @@ ${project.version} - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-1.2-api - - org.apache.hadoop @@ -146,6 +132,13 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + + org.junit.jupiter junit-jupiter-api diff --git a/hudi-aws/src/test/resources/log4j-surefire.properties b/hudi-aws/src/test/resources/log4j-surefire.properties deleted file mode 100644 index a59d4ebe2b194..0000000000000 --- a/hudi-aws/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,25 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, A1 -log4j.category.org.apache=INFO -log4j.category.org.apache.parquet.hadoop=WARN -# A1 is set to be a ConsoleAppender. -log4j.appender.A1=org.apache.log4j.ConsoleAppender -# A1 uses PatternLayout. -log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/hudi-cli/pom.xml b/hudi-cli/pom.xml index f4b743f1e2562..0910d94c82d37 100644 --- a/hudi-cli/pom.xml +++ b/hudi-cli/pom.xml @@ -200,35 +200,10 @@ - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-1.2-api - - org.apache.logging.log4j log4j-core - test - - - - org.apache.logging.log4j - log4j-api - test - - - - org.apache.logging.log4j - log4j-slf4j-impl - test + compile @@ -330,6 +305,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.junit.jupiter junit-jupiter-api diff --git a/hudi-cli/src/test/resources/log4j-surefire-quiet.properties b/hudi-cli/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index b21b5d4070c41..0000000000000 --- a/hudi-cli/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,29 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-cli/src/test/resources/log4j-surefire.properties b/hudi-cli/src/test/resources/log4j-surefire.properties deleted file mode 100644 index a59d4ebe2b194..0000000000000 --- a/hudi-cli/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,25 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, A1 -log4j.category.org.apache=INFO -log4j.category.org.apache.parquet.hadoop=WARN -# A1 is set to be a ConsoleAppender. -log4j.appender.A1=org.apache.log4j.ConsoleAppender -# A1 uses PatternLayout. -log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/hudi-client/hudi-client-common/pom.xml b/hudi-client/hudi-client-common/pom.xml index 3a4f37fdab94d..031a9ee6a3231 100644 --- a/hudi-client/hudi-client-common/pom.xml +++ b/hudi-client/hudi-client-common/pom.xml @@ -53,20 +53,6 @@ joda-time - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-1.2-api - - org.apache.parquet @@ -193,6 +179,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.junit.jupiter junit-jupiter-api diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/http/TestCallbackHttpClient.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/http/TestCallbackHttpClient.java index 8d3d00d572486..49b948dd8c0dc 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/http/TestCallbackHttpClient.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/http/TestCallbackHttpClient.java @@ -38,6 +38,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.io.IOException; +import java.util.UUID; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -68,17 +69,24 @@ public class TestCallbackHttpClient { @Mock StatusLine statusLine; + private Level initialLogLevel; + @BeforeEach void prepareAppender() { - when(appender.getName()).thenReturn("MockAppender"); + when(appender.getName()).thenReturn("MockAppender-" + UUID.randomUUID()); when(appender.isStarted()).thenReturn(true); when(appender.isStopped()).thenReturn(false); - ((Logger) LogManager.getLogger(HoodieWriteCommitHttpCallbackClient.class)).addAppender(appender); + Logger logger = (Logger) LogManager.getLogger(HoodieWriteCommitHttpCallbackClient.class); + initialLogLevel = logger.getLevel(); + logger.setLevel(Level.DEBUG); + logger.addAppender(appender); } @AfterEach void resetMocks() { - ((Logger) LogManager.getLogger(HoodieWriteCommitHttpCallbackClient.class)).removeAppender(appender); + Logger logger = (Logger) LogManager.getLogger(HoodieWriteCommitHttpCallbackClient.class); + logger.setLevel(initialLogLevel); + logger.removeAppender(appender); reset(appender, httpClient, httpResponse, statusLine); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogHttpClient.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogHttpClient.java index ebc46db00dabd..e968190522465 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogHttpClient.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogHttpClient.java @@ -43,6 +43,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.UUID; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -71,17 +72,24 @@ public class TestDatadogHttpClient { @Mock StatusLine statusLine; + private Level initialLogLevel; + @BeforeEach void prepareAppender() { - when(appender.getName()).thenReturn("MockAppender"); + when(appender.getName()).thenReturn("MockAppender-" + UUID.randomUUID()); when(appender.isStarted()).thenReturn(true); when(appender.isStopped()).thenReturn(false); - ((Logger) LogManager.getLogger(DatadogHttpClient.class)).addAppender(appender); + Logger logger = (Logger) LogManager.getLogger(DatadogHttpClient.class); + initialLogLevel = logger.getLevel(); + logger.setLevel(Level.DEBUG); + logger.addAppender(appender); } @AfterEach void resetMocks() { - ((org.apache.logging.log4j.core.Logger) LogManager.getLogger(DatadogHttpClient.class)).removeAppender(appender); + Logger logger = (Logger) LogManager.getLogger(DatadogHttpClient.class); + logger.setLevel(initialLogLevel); + logger.removeAppender(appender); reset(appender, httpClient, httpResponse, statusLine); } @@ -117,7 +125,6 @@ public void validateApiKeyShouldThrowExceptionWhenResponseNotSuccessful() { @Test public void sendPayloadShouldLogWhenRequestFailed() throws IOException { - ((Logger) LogManager.getLogger(DatadogHttpClient.class)).addAppender(appender); when(httpClient.execute(any())).thenThrow(IOException.class); DatadogHttpClient ddClient = new DatadogHttpClient(ApiSite.US, "foo", true, httpClient); @@ -130,7 +137,6 @@ public void sendPayloadShouldLogWhenRequestFailed() throws IOException { @Test public void sendPayloadShouldLogUnsuccessfulSending() { - ((Logger) LogManager.getLogger(DatadogHttpClient.class)).addAppender(appender); mockResponse(401); when(httpResponse.toString()).thenReturn("unauthorized"); @@ -144,7 +150,6 @@ public void sendPayloadShouldLogUnsuccessfulSending() { @Test public void sendPayloadShouldLogSuccessfulSending() { - ((Logger) LogManager.getLogger(DatadogHttpClient.class)).addAppender(appender); mockResponse(202); DatadogHttpClient ddClient = new DatadogHttpClient(ApiSite.US, "foo", true, httpClient); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogReporter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogReporter.java index 838b97c493fb3..4166a3e8060b3 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogReporter.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogReporter.java @@ -39,6 +39,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -78,7 +79,7 @@ public void stopShouldCloseEnclosedClient() throws IOException { @Test public void stopShouldLogWhenEnclosedClientFailToClose() throws IOException { - when(appender.getName()).thenReturn("MockAppender"); + when(appender.getName()).thenReturn("MockAppender-" + UUID.randomUUID()); when(appender.isStarted()).thenReturn(true); when(appender.isStopped()).thenReturn(false); ((Logger) LogManager.getLogger(DatadogReporter.class)).addAppender(appender); diff --git a/hudi-client/hudi-client-common/src/test/resources/log4j-surefire-quiet.properties b/hudi-client/hudi-client-common/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index 2b94ea2903067..0000000000000 --- a/hudi-client/hudi-client-common/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/hudi-client/hudi-client-common/src/test/resources/log4j-surefire.properties b/hudi-client/hudi-client-common/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 14bbb089724c8..0000000000000 --- a/hudi-client/hudi-client-common/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-client/hudi-flink-client/pom.xml b/hudi-client/hudi-flink-client/pom.xml index c3139c14b08f2..69a9ea8b647b7 100644 --- a/hudi-client/hudi-flink-client/pom.xml +++ b/hudi-client/hudi-flink-client/pom.xml @@ -143,6 +143,14 @@ javax.xml.bind * + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + @@ -184,6 +192,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.junit.jupiter junit-jupiter-api diff --git a/hudi-client/hudi-flink-client/src/main/resources/log4j-surefire.properties b/hudi-client/hudi-flink-client/src/main/resources/log4j-surefire.properties deleted file mode 100644 index 14bbb089724c8..0000000000000 --- a/hudi-client/hudi-flink-client/src/main/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-client/hudi-flink-client/src/main/resources/log4j.properties b/hudi-client/hudi-flink-client/src/main/resources/log4j.properties deleted file mode 100644 index ff268faf6363c..0000000000000 --- a/hudi-client/hudi-flink-client/src/main/resources/log4j.properties +++ /dev/null @@ -1,23 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=INFO, A1 -# A1 is set to be a ConsoleAppender. -log4j.appender.A1=org.apache.log4j.ConsoleAppender -# A1 uses PatternLayout. -log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/hudi-client/hudi-flink-client/src/test/resources/log4j-surefire.properties b/hudi-client/hudi-flink-client/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 14bbb089724c8..0000000000000 --- a/hudi-client/hudi-flink-client/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-client/hudi-java-client/pom.xml b/hudi-client/hudi-java-client/pom.xml index 6686c7a499a28..f487a4aa68cc1 100644 --- a/hudi-client/hudi-java-client/pom.xml +++ b/hudi-client/hudi-java-client/pom.xml @@ -81,6 +81,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.junit.jupiter junit-jupiter-api diff --git a/hudi-client/hudi-java-client/src/main/resources/log4j.properties b/hudi-client/hudi-java-client/src/main/resources/log4j.properties deleted file mode 100644 index ff268faf6363c..0000000000000 --- a/hudi-client/hudi-java-client/src/main/resources/log4j.properties +++ /dev/null @@ -1,23 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=INFO, A1 -# A1 is set to be a ConsoleAppender. -log4j.appender.A1=org.apache.log4j.ConsoleAppender -# A1 uses PatternLayout. -log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/hudi-client/hudi-java-client/src/test/resources/log4j-surefire.properties b/hudi-client/hudi-java-client/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 14bbb089724c8..0000000000000 --- a/hudi-client/hudi-java-client/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-client/hudi-spark-client/pom.xml b/hudi-client/hudi-spark-client/pom.xml index a2b6d54ad3867..19d49f5d33165 100644 --- a/hudi-client/hudi-spark-client/pom.xml +++ b/hudi-client/hudi-spark-client/pom.xml @@ -103,6 +103,14 @@ javax.xml.bind * + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + @@ -110,6 +118,22 @@ zookeeper ${zookeeper.version} test + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + + + + + org.slf4j + jul-to-slf4j @@ -128,6 +152,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.junit.jupiter junit-jupiter-api diff --git a/hudi-client/hudi-spark-client/src/main/resources/log4j.properties b/hudi-client/hudi-spark-client/src/main/resources/log4j.properties deleted file mode 100644 index ff268faf6363c..0000000000000 --- a/hudi-client/hudi-spark-client/src/main/resources/log4j.properties +++ /dev/null @@ -1,23 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=INFO, A1 -# A1 is set to be a ConsoleAppender. -log4j.appender.A1=org.apache.log4j.ConsoleAppender -# A1 uses PatternLayout. -log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/hudi-client/hudi-spark-client/src/test/resources/log4j-surefire-quiet.properties b/hudi-client/hudi-spark-client/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index 2b94ea2903067..0000000000000 --- a/hudi-client/hudi-spark-client/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/hudi-client/hudi-spark-client/src/test/resources/log4j-surefire.properties b/hudi-client/hudi-spark-client/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 14bbb089724c8..0000000000000 --- a/hudi-client/hudi-spark-client/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml index 74d8184924e3e..bfbfdee776b08 100644 --- a/hudi-common/pom.xml +++ b/hudi-common/pom.xml @@ -101,6 +101,28 @@ + + + org.apache.logging.log4j + log4j-1.2-api + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.slf4j + jul-to-slf4j + + + org.slf4j + slf4j-api + + com.fasterxml.jackson.core @@ -182,6 +204,13 @@ test + + org.apache.hudi + hudi-tests-common + ${project.version} + test + + org.junit.jupiter junit-jupiter-api @@ -230,6 +259,16 @@ org.apache.hbase hbase-client ${hbase.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + @@ -270,6 +309,5 @@ joda-time test - diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java index 62cc23a99c943..9fa96216f4d53 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/TestPriorityBasedFileSystemView.java @@ -31,10 +31,11 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.http.client.HttpResponseException; -import org.apache.log4j.AppenderSkeleton; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.log4j.spi.LoggingEvent; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.Logger; +import org.apache.logging.log4j.core.appender.AbstractAppender; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -45,6 +46,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.UUID; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -112,8 +114,9 @@ public void testGetLatestBaseFiles() { @Test public void testBadRequestExceptionWithPrimary() { final TestLogAppender appender = new TestLogAppender(); - final Logger logger = Logger.getRootLogger(); + final Logger logger = (Logger) LogManager.getLogger(PriorityBasedFileSystemView.class); try { + appender.start(); logger.addAppender(appender); Stream actual; Stream expected = testBaseFileStream; @@ -123,10 +126,10 @@ public void testBadRequestExceptionWithPrimary() { when(secondary.getLatestBaseFiles()).thenReturn(testBaseFileStream); actual = fsView.getLatestBaseFiles(); assertEquals(expected, actual); - final List logs = appender.getLog(); - final LoggingEvent firstLogEntry = logs.get(0); + final List logs = appender.getLog(); + final LogEvent firstLogEntry = logs.get(0); assertEquals(firstLogEntry.getLevel(), Level.WARN); - assertTrue(((String)firstLogEntry.getMessage()).contains("Got error running preferred function. Likely due to another " + assertTrue((firstLogEntry.getMessage().getFormattedMessage()).contains("Got error running preferred function. Likely due to another " + "concurrent writer in progress. Trying secondary")); } finally { logger.removeAppender(appender); @@ -665,25 +668,20 @@ public void testGetSecondaryView() { assertEquals(secondary, fsView.getSecondaryView()); } - class TestLogAppender extends AppenderSkeleton { - private final List log = new ArrayList(); + class TestLogAppender extends AbstractAppender { + private final List log = new ArrayList<>(); - @Override - public boolean requiresLayout() { - return false; - } - - @Override - protected void append(final LoggingEvent loggingEvent) { - log.add(loggingEvent); + protected TestLogAppender() { + super(UUID.randomUUID().toString(), null, null, false, null); } @Override - public void close() { + public void append(LogEvent event) { + log.add(event); } - public List getLog() { - return new ArrayList(log); + public List getLog() { + return new ArrayList(log); } } } diff --git a/hudi-common/src/test/resources/log4j-surefire-quiet.properties b/hudi-common/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index ca0a50c84270c..0000000000000 --- a/hudi-common/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-common/src/test/resources/log4j-surefire.properties b/hudi-common/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 31841a6a3a1ec..0000000000000 --- a/hudi-common/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/hudi-examples/hudi-examples-flink/pom.xml b/hudi-examples/hudi-examples-flink/pom.xml index a9092411d9dc0..a69107b1f1630 100644 --- a/hudi-examples/hudi-examples-flink/pom.xml +++ b/hudi-examples/hudi-examples-flink/pom.xml @@ -256,7 +256,12 @@ - + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.junit.jupiter diff --git a/hudi-examples/hudi-examples-flink/src/test/resources/log4j-surefire-quiet.properties b/hudi-examples/hudi-examples-flink/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index 2b94ea2903067..0000000000000 --- a/hudi-examples/hudi-examples-flink/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/hudi-examples/hudi-examples-flink/src/test/resources/log4j-surefire.properties b/hudi-examples/hudi-examples-flink/src/test/resources/log4j-surefire.properties deleted file mode 100644 index ac2807e332775..0000000000000 --- a/hudi-examples/hudi-examples-flink/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=INFO, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=INFO -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-examples/hudi-examples-java/pom.xml b/hudi-examples/hudi-examples-java/pom.xml index 82340763689e1..2f4fa618016ac 100644 --- a/hudi-examples/hudi-examples-java/pom.xml +++ b/hudi-examples/hudi-examples-java/pom.xml @@ -152,6 +152,12 @@ mockito-junit-jupiter test + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.apache.parquet parquet-hadoop diff --git a/hudi-examples/hudi-examples-spark/pom.xml b/hudi-examples/hudi-examples-spark/pom.xml index c7b774bc0b262..1b5e9a1455aa5 100644 --- a/hudi-examples/hudi-examples-spark/pom.xml +++ b/hudi-examples/hudi-examples-spark/pom.xml @@ -236,6 +236,11 @@ + + + org.apache.logging.log4j + log4j-core + @@ -265,6 +270,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.apache.hudi hudi-client-common diff --git a/hudi-examples/hudi-examples-spark/src/test/resources/log4j-surefire-quiet.properties b/hudi-examples/hudi-examples-spark/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index 2b94ea2903067..0000000000000 --- a/hudi-examples/hudi-examples-spark/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/hudi-examples/hudi-examples-spark/src/test/resources/log4j-surefire.properties b/hudi-examples/hudi-examples-spark/src/test/resources/log4j-surefire.properties deleted file mode 100644 index ac2807e332775..0000000000000 --- a/hudi-examples/hudi-examples-spark/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=INFO, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=INFO -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-flink-datasource/hudi-flink/pom.xml b/hudi-flink-datasource/hudi-flink/pom.xml index f6bd3a00acb43..ae853a4851aa0 100644 --- a/hudi-flink-datasource/hudi-flink/pom.xml +++ b/hudi-flink-datasource/hudi-flink/pom.xml @@ -291,7 +291,12 @@ - + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.junit.jupiter diff --git a/hudi-flink-datasource/hudi-flink/src/test/resources/log4j-surefire-quiet.properties b/hudi-flink-datasource/hudi-flink/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index 40171af5f4083..0000000000000 --- a/hudi-flink-datasource/hudi-flink/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,29 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/hudi-flink-datasource/hudi-flink/src/test/resources/log4j-surefire.properties b/hudi-flink-datasource/hudi-flink/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 6108218dc582a..0000000000000 --- a/hudi-flink-datasource/hudi-flink/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=INFO, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=INFO -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-flink-datasource/hudi-flink1.13.x/pom.xml b/hudi-flink-datasource/hudi-flink1.13.x/pom.xml index 46a8db48e1b89..8cb923b4e1645 100644 --- a/hudi-flink-datasource/hudi-flink1.13.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink1.13.x/pom.xml @@ -83,7 +83,12 @@ - + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.junit.jupiter diff --git a/hudi-flink-datasource/hudi-flink1.14.x/pom.xml b/hudi-flink-datasource/hudi-flink1.14.x/pom.xml index ba9e18c4b60a9..cef0e2b8392a0 100644 --- a/hudi-flink-datasource/hudi-flink1.14.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink1.14.x/pom.xml @@ -101,7 +101,12 @@ - + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.junit.jupiter diff --git a/hudi-flink-datasource/hudi-flink1.15.x/pom.xml b/hudi-flink-datasource/hudi-flink1.15.x/pom.xml index 713ae27c55bfc..a9c2519736fde 100644 --- a/hudi-flink-datasource/hudi-flink1.15.x/pom.xml +++ b/hudi-flink-datasource/hudi-flink1.15.x/pom.xml @@ -101,7 +101,12 @@ - + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.junit.jupiter diff --git a/hudi-gcp/pom.xml b/hudi-gcp/pom.xml index a2b50b94cb0e7..3a93fa57384e6 100644 --- a/hudi-gcp/pom.xml +++ b/hudi-gcp/pom.xml @@ -61,20 +61,6 @@ See https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google google-cloud-bigquery - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-1.2-api - - org.apache.parquet parquet-avro @@ -86,6 +72,13 @@ See https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google hadoop-common + + org.apache.hudi + hudi-tests-common + ${project.version} + test + + org.junit.jupiter junit-jupiter-api diff --git a/hudi-gcp/src/test/resources/log4j-surefire.properties b/hudi-gcp/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 7914f0a78273b..0000000000000 --- a/hudi-gcp/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,29 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=INFO - -# A1 is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# A1 uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-hadoop-mr/pom.xml b/hudi-hadoop-mr/pom.xml index 0d288b39099cc..720b17654d89d 100644 --- a/hudi-hadoop-mr/pom.xml +++ b/hudi-hadoop-mr/pom.xml @@ -75,6 +75,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.apache.hudi hudi-common diff --git a/hudi-hadoop-mr/src/test/resources/log4j-surefire-quiet.properties b/hudi-hadoop-mr/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index b21b5d4070c41..0000000000000 --- a/hudi-hadoop-mr/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,29 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-hadoop-mr/src/test/resources/log4j-surefire.properties b/hudi-hadoop-mr/src/test/resources/log4j-surefire.properties deleted file mode 100644 index fe87e2deeb071..0000000000000 --- a/hudi-hadoop-mr/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/hudi-integ-test/pom.xml b/hudi-integ-test/pom.xml index 1b371efaa839c..ca1e306a04ca0 100644 --- a/hudi-integ-test/pom.xml +++ b/hudi-integ-test/pom.xml @@ -164,35 +164,10 @@ - - org.apache.logging.log4j - log4j-api - org.apache.logging.log4j log4j-core - - - org.apache.logging.log4j - log4j-1.2-api - - - - org.apache.logging.log4j - log4j-core - test - - - - org.apache.logging.log4j - log4j-api - test - - - - org.apache.logging.log4j - log4j-slf4j-impl - test + compile @@ -204,6 +179,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.apache.hudi hudi-common @@ -535,24 +516,9 @@ - org.scalatest - scalatest-maven-plugin - 1.0 - - ${skipUTs} - ${project.build.directory}/surefire-reports - . - TestSuite.txt - - - - test - - test - - - - + org.scalatest + scalatest-maven-plugin + org.scalastyle scalastyle-maven-plugin diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java index 36000e9cb86e8..148fc862a1a1b 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java @@ -200,7 +200,6 @@ private void setupDemo() throws Exception { private void ingestFirstBatchAndHiveSync() throws Exception { List cmds = CollectionUtils.createImmutableList( "spark-submit" - + " --conf \'spark.executor.extraJavaOptions=-Dlog4jspark.root.logger=WARN,console\'" + " --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE + " --table-type COPY_ON_WRITE " + " --base-file-format " + baseFileFormat.toString() @@ -219,7 +218,6 @@ private void ingestFirstBatchAndHiveSync() throws Exception { + " --partition-value-extractor org.apache.hudi.hive.SlashEncodedDayPartitionValueExtractor" + " --partitioned-by dt", ("spark-submit" - + " --conf \'spark.executor.extraJavaOptions=-Dlog4jspark.root.logger=WARN,console\'" + " --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE + " --table-type MERGE_ON_READ " + " --base-file-format " + baseFileFormat.toString() @@ -309,7 +307,6 @@ private void ingestSecondBatchAndHiveSync() throws Exception { List cmds = CollectionUtils.createImmutableList( ("hdfs dfs -copyFromLocal -f " + INPUT_BATCH_PATH2 + " " + HDFS_BATCH_PATH2), ("spark-submit" - + " --conf \'spark.executor.extraJavaOptions=-Dlog4jspark.root.logger=WARN,console\'" + " --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE + " --table-type COPY_ON_WRITE " + " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts " @@ -318,7 +315,6 @@ private void ingestSecondBatchAndHiveSync() throws Exception { + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider " + String.format(HIVE_SYNC_CMD_FMT, "dt", COW_TABLE_NAME)), ("spark-submit" - + " --conf \'spark.executor.extraJavaOptions=-Dlog4jspark.root.logger=WARN,console\'" + " --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE + " --table-type MERGE_ON_READ " + " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts " diff --git a/hudi-integ-test/src/test/resources/log4j-surefire-quiet.properties b/hudi-integ-test/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index 61fbf78d1dffb..0000000000000 --- a/hudi-integ-test/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=ERROR, CONSOLE -log4j.logger.org.apache.hudi=ERROR -log4j.category.org.apache.spark=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=ERROR -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-integ-test/src/test/resources/log4j-surefire.properties b/hudi-integ-test/src/test/resources/log4j-surefire.properties deleted file mode 100644 index fe87e2deeb071..0000000000000 --- a/hudi-integ-test/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/hudi-kafka-connect/pom.xml b/hudi-kafka-connect/pom.xml index 5be572a0690e8..304afddd27272 100644 --- a/hudi-kafka-connect/pom.xml +++ b/hudi-kafka-connect/pom.xml @@ -170,17 +170,10 @@ - - org.apache.logging.log4j - log4j-api - org.apache.logging.log4j log4j-core - - - org.apache.logging.log4j - log4j-1.2-api + compile @@ -222,6 +215,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.apache.hudi hudi-common diff --git a/hudi-kafka-connect/src/main/resources/log4j.properties b/hudi-kafka-connect/src/main/resources/log4j.properties deleted file mode 100644 index ff268faf6363c..0000000000000 --- a/hudi-kafka-connect/src/main/resources/log4j.properties +++ /dev/null @@ -1,23 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=INFO, A1 -# A1 is set to be a ConsoleAppender. -log4j.appender.A1=org.apache.log4j.ConsoleAppender -# A1 uses PatternLayout. -log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/hudi-kafka-connect/src/test/resources/log4j-surefire-quiet.properties b/hudi-kafka-connect/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index ca0a50c84270c..0000000000000 --- a/hudi-kafka-connect/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-kafka-connect/src/test/resources/log4j-surefire.properties b/hudi-kafka-connect/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 14bbb089724c8..0000000000000 --- a/hudi-kafka-connect/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-spark-datasource/hudi-spark-common/pom.xml b/hudi-spark-datasource/hudi-spark-common/pom.xml index 1e517679185da..27bb1405b97a5 100644 --- a/hudi-spark-datasource/hudi-spark-common/pom.xml +++ b/hudi-spark-datasource/hudi-spark-common/pom.xml @@ -212,6 +212,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.apache.hudi hudi-client-common diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/resources/log4j-surefire-quiet.properties b/hudi-spark-datasource/hudi-spark-common/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index ca0a50c84270c..0000000000000 --- a/hudi-spark-datasource/hudi-spark-common/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-spark-datasource/hudi-spark-common/src/test/resources/log4j-surefire.properties b/hudi-spark-datasource/hudi-spark-common/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 14bbb089724c8..0000000000000 --- a/hudi-spark-datasource/hudi-spark-common/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-spark-datasource/hudi-spark/pom.xml b/hudi-spark-datasource/hudi-spark/pom.xml index f5bde2c5aabff..035c099c3a423 100644 --- a/hudi-spark-datasource/hudi-spark/pom.xml +++ b/hudi-spark-datasource/hudi-spark/pom.xml @@ -135,21 +135,6 @@ org.scalatest scalatest-maven-plugin - 1.0 - - ${skipUTs} - ${project.build.directory}/surefire-reports - . - TestSuite.txt - - - - test - - test - - - org.scalastyle @@ -254,20 +239,6 @@ - - - org.apache.logging.log4j - log4j-api - - - org.apache.logging.log4j - log4j-core - - - org.apache.logging.log4j - log4j-1.2-api - - com.fasterxml.jackson.core @@ -445,6 +416,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.apache.hudi hudi-client-common diff --git a/hudi-spark-datasource/hudi-spark/src/test/resources/log4j-surefire-quiet.properties b/hudi-spark-datasource/hudi-spark/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index ca0a50c84270c..0000000000000 --- a/hudi-spark-datasource/hudi-spark/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-spark-datasource/hudi-spark/src/test/resources/log4j-surefire.properties b/hudi-spark-datasource/hudi-spark/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 14bbb089724c8..0000000000000 --- a/hudi-spark-datasource/hudi-spark/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-spark-datasource/hudi-spark2/pom.xml b/hudi-spark-datasource/hudi-spark2/pom.xml index d01152f495a18..8bdab01ba1c51 100644 --- a/hudi-spark-datasource/hudi-spark2/pom.xml +++ b/hudi-spark-datasource/hudi-spark2/pom.xml @@ -204,6 +204,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.apache.hudi hudi-client-common diff --git a/hudi-spark-datasource/hudi-spark2/src/test/resources/log4j-surefire-quiet.properties b/hudi-spark-datasource/hudi-spark2/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index ca0a50c84270c..0000000000000 --- a/hudi-spark-datasource/hudi-spark2/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-spark-datasource/hudi-spark2/src/test/resources/log4j-surefire.properties b/hudi-spark-datasource/hudi-spark2/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 14bbb089724c8..0000000000000 --- a/hudi-spark-datasource/hudi-spark2/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-spark-datasource/hudi-spark3-common/pom.xml b/hudi-spark-datasource/hudi-spark3-common/pom.xml index 760ca1af5af54..19f73c4ca625b 100644 --- a/hudi-spark-datasource/hudi-spark3-common/pom.xml +++ b/hudi-spark-datasource/hudi-spark3-common/pom.xml @@ -200,6 +200,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.apache.hudi hudi-client-common diff --git a/hudi-spark-datasource/hudi-spark3.1.x/pom.xml b/hudi-spark-datasource/hudi-spark3.1.x/pom.xml index 10e70c5c61fb9..d4b428d258073 100644 --- a/hudi-spark-datasource/hudi-spark3.1.x/pom.xml +++ b/hudi-spark-datasource/hudi-spark3.1.x/pom.xml @@ -215,6 +215,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.apache.hudi hudi-client-common diff --git a/hudi-spark-datasource/hudi-spark3.2.x/src/test/resources/log4j-surefire-quiet.properties b/hudi-spark-datasource/hudi-spark3.2.x/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index ca0a50c84270c..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.2.x/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-spark-datasource/hudi-spark3.2.x/src/test/resources/log4j-surefire.properties b/hudi-spark-datasource/hudi-spark3.2.x/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 14bbb089724c8..0000000000000 --- a/hudi-spark-datasource/hudi-spark3.2.x/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-spark-datasource/hudi-spark3.3.x/pom.xml b/hudi-spark-datasource/hudi-spark3.3.x/pom.xml index abb25bf61b067..8818c63de1858 100644 --- a/hudi-spark-datasource/hudi-spark3.3.x/pom.xml +++ b/hudi-spark-datasource/hudi-spark3.3.x/pom.xml @@ -255,6 +255,12 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + org.apache.hudi diff --git a/hudi-sync/hudi-adb-sync/pom.xml b/hudi-sync/hudi-adb-sync/pom.xml index 1e6841767b055..fc2f914e8ebf7 100644 --- a/hudi-sync/hudi-adb-sync/pom.xml +++ b/hudi-sync/hudi-adb-sync/pom.xml @@ -97,17 +97,10 @@ - - org.apache.logging.log4j - log4j-api - org.apache.logging.log4j log4j-core - - - org.apache.logging.log4j - log4j-1.2-api + compile @@ -126,6 +119,13 @@ ${hive.version} + + org.apache.hudi + hudi-tests-common + ${project.version} + test + + org.junit.jupiter junit-jupiter-api diff --git a/hudi-sync/hudi-adb-sync/src/test/resources/log4j-surefire-quiet.properties b/hudi-sync/hudi-adb-sync/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index b21b5d4070c41..0000000000000 --- a/hudi-sync/hudi-adb-sync/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,29 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-sync/hudi-adb-sync/src/test/resources/log4j-surefire.properties b/hudi-sync/hudi-adb-sync/src/test/resources/log4j-surefire.properties deleted file mode 100644 index fe87e2deeb071..0000000000000 --- a/hudi-sync/hudi-adb-sync/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/hudi-sync/hudi-datahub-sync/pom.xml b/hudi-sync/hudi-datahub-sync/pom.xml index 040a53990f7ab..08b13b55fbb8c 100644 --- a/hudi-sync/hudi-datahub-sync/pom.xml +++ b/hudi-sync/hudi-datahub-sync/pom.xml @@ -69,17 +69,10 @@ - - org.apache.logging.log4j - log4j-api - org.apache.logging.log4j log4j-core - - - org.apache.logging.log4j - log4j-1.2-api + compile @@ -99,6 +92,13 @@ ${project.version} + + org.apache.hudi + hudi-tests-common + ${project.version} + test + + org.junit.jupiter junit-jupiter-api diff --git a/hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire-quiet.properties b/hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index 78d6cfe849883..0000000000000 --- a/hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,29 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=ERROR, CONSOLE -log4j.logger.org.apache.hudi=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire.properties b/hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 9782bf7d9c378..0000000000000 --- a/hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,29 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=INFO - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-sync/hudi-hive-sync/pom.xml b/hudi-sync/hudi-hive-sync/pom.xml index 8641580977171..738bd218e9429 100644 --- a/hudi-sync/hudi-hive-sync/pom.xml +++ b/hudi-sync/hudi-hive-sync/pom.xml @@ -54,17 +54,10 @@ - - org.apache.logging.log4j - log4j-api - org.apache.logging.log4j log4j-core - - - org.apache.logging.log4j - log4j-1.2-api + compile @@ -160,8 +153,9 @@ - org.apache.logging.log4j - log4j-core + org.apache.hudi + hudi-tests-common + ${project.version} test diff --git a/hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire-quiet.properties b/hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index b21b5d4070c41..0000000000000 --- a/hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,29 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire.properties b/hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire.properties deleted file mode 100644 index fe87e2deeb071..0000000000000 --- a/hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/hudi-sync/hudi-sync-common/pom.xml b/hudi-sync/hudi-sync-common/pom.xml index f4f15c478ca0f..84d20f893fa9c 100644 --- a/hudi-sync/hudi-sync-common/pom.xml +++ b/hudi-sync/hudi-sync-common/pom.xml @@ -52,8 +52,9 @@ - org.apache.logging.log4j - log4j-core + org.apache.hudi + hudi-tests-common + ${project.version} test diff --git a/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire-quiet.properties b/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index b21b5d4070c41..0000000000000 --- a/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,29 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire.properties b/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire.properties deleted file mode 100644 index fe87e2deeb071..0000000000000 --- a/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/hudi-tests-common/pom.xml b/hudi-tests-common/pom.xml new file mode 100644 index 0000000000000..e66569c64de39 --- /dev/null +++ b/hudi-tests-common/pom.xml @@ -0,0 +1,71 @@ + + + + + hudi + org.apache.hudi + 0.13.0-SNAPSHOT + + + 4.0.0 + hudi-tests-common + + + ${project.parent.basedir} + + + + + + + src/main/resources + + + + + + + org.apache.rat + apache-rat-plugin + + + + + + + + org.apache.logging.log4j + log4j-core + compile + + + + org.apache.logging.log4j + log4j-slf4j-impl + compile + + + org.slf4j + jul-to-slf4j + compile + + + + org.junit.jupiter + junit-jupiter-engine + + + \ No newline at end of file diff --git a/hudi-client/hudi-client-common/src/main/resources/log4j.properties b/hudi-tests-common/src/main/resources/log4j2-surefire-quiet.properties similarity index 64% rename from hudi-client/hudi-client-common/src/main/resources/log4j.properties rename to hudi-tests-common/src/main/resources/log4j2-surefire-quiet.properties index ff268faf6363c..287af214fb1d4 100644 --- a/hudi-client/hudi-client-common/src/main/resources/log4j.properties +++ b/hudi-tests-common/src/main/resources/log4j2-surefire-quiet.properties @@ -15,9 +15,19 @@ # See the License for the specific language governing permissions and # limitations under the License. ### -log4j.rootLogger=INFO, A1 -# A1 is set to be a ConsoleAppender. -log4j.appender.A1=org.apache.log4j.ConsoleAppender -# A1 uses PatternLayout. -log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n +status = warn +name = HudiConsoleLog + +# Console appender configuration +appender.console.type = Console +appender.console.name = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %-4r [%t] %-5p %c %x - %m%n + +# Root logger level +rootLogger.level = warn +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = CONSOLE + +logger.hbase.name = org.apache.hadoop.hbase +logger.hbase.level = error \ No newline at end of file diff --git a/hudi-gcp/src/test/resources/log4j-surefire-quiet.properties b/hudi-tests-common/src/main/resources/log4j2-surefire.properties similarity index 58% rename from hudi-gcp/src/test/resources/log4j-surefire-quiet.properties rename to hudi-tests-common/src/main/resources/log4j2-surefire.properties index 78d6cfe849883..6b6b2fa5e5322 100644 --- a/hudi-gcp/src/test/resources/log4j-surefire-quiet.properties +++ b/hudi-tests-common/src/main/resources/log4j2-surefire.properties @@ -15,15 +15,23 @@ # See the License for the specific language governing permissions and # limitations under the License. ### -log4j.rootLogger=ERROR, CONSOLE -log4j.logger.org.apache.hudi=ERROR +status = warn +name = HudiConsoleLog -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL +# Console appender configuration +appender.console.type = Console +appender.console.name = CONSOLE +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %-4r [%t] %-5p %c %x - %m%n + +# Root logger level +rootLogger.level = warn +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = CONSOLE + +logger.apache.name = org.apache +logger.apache.level = info +logger.hudi.name = org.apache.hudi +logger.hudi.level = debug +logger.hbase.name = org.apache.hadoop.hbase +logger.hbase.level = error \ No newline at end of file diff --git a/hudi-timeline-service/pom.xml b/hudi-timeline-service/pom.xml index 30162a404d15a..ff5b8dfe7add4 100644 --- a/hudi-timeline-service/pom.xml +++ b/hudi-timeline-service/pom.xml @@ -76,17 +76,10 @@ - - org.apache.logging.log4j - log4j-api - org.apache.logging.log4j log4j-core - - - org.apache.logging.log4j - log4j-1.2-api + compile @@ -179,6 +172,13 @@ + + org.apache.hudi + hudi-tests-common + ${project.version} + test + + org.apache.hudi hudi-common diff --git a/hudi-timeline-service/src/test/resources/log4j-surefire-quiet.properties b/hudi-timeline-service/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index b21b5d4070c41..0000000000000 --- a/hudi-timeline-service/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,29 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-timeline-service/src/test/resources/log4j-surefire.properties b/hudi-timeline-service/src/test/resources/log4j-surefire.properties deleted file mode 100644 index fe87e2deeb071..0000000000000 --- a/hudi-timeline-service/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/hudi-utilities/pom.xml b/hudi-utilities/pom.xml index 322297dfdadf1..cbbe4e76a343e 100644 --- a/hudi-utilities/pom.xml +++ b/hudi-utilities/pom.xml @@ -172,23 +172,10 @@ - - org.apache.logging.log4j - log4j-api - org.apache.logging.log4j log4j-core - - - org.apache.logging.log4j - log4j-1.2-api - - - - org.slf4j - slf4j-api - ${slf4j.version} + compile @@ -432,6 +419,12 @@ test-jar test + + org.apache.hudi + hudi-tests-common + ${project.version} + test + @@ -497,11 +490,5 @@ junit-platform-commons test - - - org.apache.logging.log4j - log4j-core - test - diff --git a/hudi-utilities/src/test/resources/log4j-surefire-quiet.properties b/hudi-utilities/src/test/resources/log4j-surefire-quiet.properties deleted file mode 100644 index ca0a50c84270c..0000000000000 --- a/hudi-utilities/src/test/resources/log4j-surefire-quiet.properties +++ /dev/null @@ -1,30 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL diff --git a/hudi-utilities/src/test/resources/log4j-surefire.properties b/hudi-utilities/src/test/resources/log4j-surefire.properties deleted file mode 100644 index 31841a6a3a1ec..0000000000000 --- a/hudi-utilities/src/test/resources/log4j-surefire.properties +++ /dev/null @@ -1,31 +0,0 @@ -### -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### -log4j.rootLogger=WARN, CONSOLE -log4j.logger.org.apache=INFO -log4j.logger.org.apache.hudi=DEBUG -log4j.logger.org.apache.hadoop.hbase=ERROR - -# CONSOLE is set to be a ConsoleAppender. -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -# CONSOLE uses PatternLayout. -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n -log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter -log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true -log4j.appender.CONSOLE.filter.a.LevelMin=WARN -log4j.appender.CONSOLE.filter.a.LevelMax=FATAL \ No newline at end of file diff --git a/packaging/hudi-timeline-server-bundle/pom.xml b/packaging/hudi-timeline-server-bundle/pom.xml index 25af0d2cffa4f..a8a0253a4adeb 100644 --- a/packaging/hudi-timeline-server-bundle/pom.xml +++ b/packaging/hudi-timeline-server-bundle/pom.xml @@ -43,17 +43,10 @@ - - org.apache.logging.log4j - log4j-api - org.apache.logging.log4j log4j-core - - - org.apache.logging.log4j - log4j-1.2-api + compile diff --git a/pom.xml b/pom.xml index 54be08f6c03ba..4d149e430ee81 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,7 @@ hudi-kafka-connect packaging/hudi-flink-bundle packaging/hudi-kafka-connect-bundle + hudi-tests-common @@ -106,9 +107,8 @@ 5.7.2 1.7.2 3.3.3 - 2.17.2 2.17.2 - 1.7.30 + 1.7.36 2.9.9 2.10.1 org.apache.hive @@ -157,7 +157,7 @@ 3.3.1 3.0.1 3.1.0 - file://${project.basedir}/src/test/resources/log4j-surefire.properties + log4j2-surefire.properties 0.12.0 9.4.15.v20190215 3.1.0-incubating @@ -353,6 +353,37 @@ false + + org.apache.maven.plugins + maven-enforcer-plugin + ${maven-enforcer-plugin.version} + + + enforce-logging + + enforce + + + + + + org.slf4j:slf4j-log4j12 + org.sl4fj:slf4j-simple + org.sl4fj:slf4j-jdk14 + org.sl4fj:slf4j-nop + org.sl4fj:slf4j-jcl + log4j:log4j + ch.qos.logback:logback-classic + + + org.slf4j:slf4j-simple:*:*:test + + + + + + + org.jacoco jacoco-maven-plugin @@ -379,13 +410,33 @@ @{argLine} false - - ${surefire-log4j.file} - + ${surefire-log4j.file} false + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${skipUTs} + ${project.build.directory}/surefire-reports + . + TestSuite.txt + + ${surefire-log4j.file} + + + + + test + + test + + + + org.apache.maven.plugins maven-jar-plugin @@ -531,12 +582,28 @@ org.apache.logging.log4j log4j-core ${log4j2.version} + provided org.apache.logging.log4j log4j-1.2-api ${log4j2.version} + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j2.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.slf4j + jul-to-slf4j + ${slf4j.version} + @@ -609,6 +676,12 @@ parquet-avro ${parquet.version} provided + + + log4j + log4j + + @@ -617,6 +690,16 @@ spark-core_${scala.binary.version} ${spark.version} provided + + + org.slf4j + * + + + log4j + log4j + + org.apache.spark @@ -643,6 +726,16 @@ tests ${spark.version} test + + + org.slf4j + * + + + log4j + log4j + + org.apache.spark @@ -794,6 +887,10 @@ javax.xml.bind jaxb-api + + org.slf4j + slf4j-log4j12 + @@ -810,6 +907,14 @@ javax.xml.bind jaxb-api + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + @@ -817,12 +922,28 @@ hadoop-hdfs ${hadoop.version} provided + + + log4j + log4j + + org.apache.hadoop hadoop-auth ${hadoop.version} provided + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + org.apache.hadoop @@ -834,6 +955,14 @@ javax.xml.bind jaxb-api + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + @@ -846,6 +975,14 @@ javax.xml.bind jaxb-api + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + @@ -854,6 +991,12 @@ tests test ${hadoop.version} + + + log4j + log4j + + org.apache.hadoop @@ -869,6 +1012,14 @@ javax.xml.bind jaxb-api + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + @@ -903,6 +1054,10 @@ org.slf4j slf4j-log4j12 + + log4j + log4j + @@ -943,6 +1098,14 @@ org.eclipse.jetty.aggregate * + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + @@ -979,6 +1142,10 @@ org.eclipse.jetty.aggregate * + + log4j + log4j + @@ -999,6 +1166,14 @@ org.apache.logging.log4j * + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + @@ -1024,6 +1199,14 @@ org.apache.logging.log4j * + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + @@ -1068,6 +1251,10 @@ org.apache.logging.log4j * + + log4j + log4j + @@ -1088,6 +1275,16 @@ org.apache.curator curator-framework ${zk-curator.version} + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + @@ -1186,28 +1383,6 @@ - - - org.apache.logging.log4j - log4j-core - ${log4j.test.version} - test - - - - org.apache.logging.log4j - log4j-api - ${log4j.test.version} - test - - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.test.version} - test - - @@ -1303,8 +1478,7 @@ - file://${project.basedir}/src/test/resources/log4j-surefire-quiet.properties - + log4j2-surefire-quiet.properties