Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions core/src/test/scala/org/apache/spark/SparkFunSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -189,12 +189,14 @@ abstract class SparkFunSuite
}
}

class LogAppender(maxEvents: Int = 100) extends AppenderSkeleton {
class LogAppender(msg: String = "", maxEvents: Int = 100) extends AppenderSkeleton {
val loggingEvents = new ArrayBuffer[LoggingEvent]()

override def append(loggingEvent: LoggingEvent): Unit = {
if (loggingEvents.size >= maxEvents) {
throw new IllegalStateException(s"Number of logging event reached the limit: $maxEvents")
val loggingInfo = if (msg == "") "." else s" while logging $msg."
throw new IllegalStateException(
s"Number of events reached the limit of $maxEvents$loggingInfo")
}
loggingEvents.append(loggingEvent)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1499,7 +1499,7 @@ abstract class AvroSuite extends QueryTest with SharedSparkSession {
}

test("log a warning of ignoreExtension deprecation") {
val logAppender = new LogAppender
val logAppender = new LogAppender("deprecated Avro option 'ignoreExtension'")
withTempPath { dir =>
Seq(("a", 1, 2), ("b", 1, 2), ("c", 2, 1), ("d", 2, 1))
.toDF("value", "p1", "p2")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ class ResolveHintsSuite extends AnalysisTest {
}

test("log warnings for invalid hints") {
val logAppender = new LogAppender
val logAppender = new LogAppender("invalid hints")
withLogAppender(logAppender) {
checkAnalysis(
UnresolvedHint("unknown_hint", Seq("TaBlE"), table("TaBlE")),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -519,7 +519,7 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
}

test("SPARK-25113: should log when there exists generated methods above HugeMethodLimit") {
val appender = new LogAppender
val appender = new LogAppender("huge method limit")
withLogAppender(appender, loggerName = Some(classOf[CodeGenerator[_, _]].getName)) {
val x = 42
val expr = HugeCodeIntExpression(x)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ class OptimizerLoggingSuite extends PlanTest {
}

private def verifyLog(expectedLevel: Level, expectedRulesOrBatches: Seq[String]): Unit = {
val logAppender = new LogAppender
val logAppender = new LogAppender("optimizer rules")
withLogAppender(logAppender,
loggerName = Some(Optimize.getClass.getName.dropRight(1)), level = Some(Level.TRACE)) {
val input = LocalRelation('a.int, 'b.string, 'c.double)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ class JoinHintSuite extends PlanTest with SharedSparkSession with AdaptiveSparkP
df: => DataFrame,
expectedHints: Seq[JoinHint],
warnings: Seq[String]): Unit = {
val logAppender = new LogAppender
val logAppender = new LogAppender("join hints")
withLogAppender(logAppender) {
verifyJoinHint(df, expectedHints)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1761,7 +1761,7 @@ class CSVSuite extends QueryTest with SharedSparkSession with TestCsvData {
}

test("SPARK-23786: warning should be printed if CSV header doesn't conform to schema") {
val testAppender1 = new LogAppender
val testAppender1 = new LogAppender("CSV header matches to schema")
withLogAppender(testAppender1) {
val ds = Seq("columnA,columnB", "1.0,1000.0").toDS()
val ischema = new StructType().add("columnB", DoubleType).add("columnA", DoubleType)
Expand All @@ -1771,7 +1771,7 @@ class CSVSuite extends QueryTest with SharedSparkSession with TestCsvData {
assert(testAppender1.loggingEvents
.exists(msg => msg.getRenderedMessage.contains("CSV header does not conform to the schema")))

val testAppender2 = new LogAppender
val testAppender2 = new LogAppender("CSV header matches to schema w/ enforceSchema")
withLogAppender(testAppender2) {
withTempPath { path =>
val oschema = new StructType().add("f1", DoubleType).add("f2", DoubleType)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,7 @@ class SQLConfSuite extends QueryTest with SharedSparkSession {
}

test("log deprecation warnings") {
val logAppender = new LogAppender
val logAppender = new LogAppender("deprecated SQL configs")
def check(config: String): Unit = {
assert(logAppender.loggingEvents.exists(
e => e.getLevel == Level.WARN &&
Expand Down