diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java index bbd8440448fd6..6f5a11ad6657f 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkTempViewProvider.java @@ -20,8 +20,6 @@ import org.apache.hudi.exception.HoodieException; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; @@ -31,27 +29,34 @@ import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructType; +import org.springframework.shell.support.logging.HandlerUtils; import java.util.List; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; public class SparkTempViewProvider implements TempViewProvider { - private static final Logger LOG = LogManager.getLogger(SparkTempViewProvider.class); + private static final Logger LOG = HandlerUtils.getLogger(SparkTempViewProvider.class); private JavaSparkContext jsc; private SQLContext sqlContext; public SparkTempViewProvider(String appName) { try { + Handler handler = LOG.getParent().getHandlers()[0]; SparkConf sparkConf = new SparkConf().setAppName(appName) .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer").setMaster("local[8]"); jsc = new JavaSparkContext(sparkConf); - jsc.setLogLevel("ERROR"); - sqlContext = new SQLContext(jsc); + if (handler != null) { + LOG.getParent().removeHandler(LOG.getParent().getHandlers()[0]); + LOG.getParent().addHandler(handler); + } } catch (Throwable ex) { // log full stack trace and rethrow. Without this its difficult to debug failures, if any - LOG.error("unable to initialize spark context ", ex); + LOG.log(Level.WARNING, "unable to initialize spark context ", ex); throw new HoodieException(ex); } } @@ -90,7 +95,7 @@ public void createOrReplace(String tableName, List headers, List