diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExpressionUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala similarity index 99% rename from sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExpressionUtils.scala rename to sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala index 22fd5b041b49..bbe27831f01d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExpressionUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVExprUtils.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.catalyst.csv -object CSVExpressionUtils { +object CSVExprUtils { /** * Filter ignorable rows for CSV iterator (lines empty and starting with `comment`). * This is currently being used in CSV reading path and CSV schema inference. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVHeaderChecker.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVHeaderChecker.scala index 4665537dcdb7..c39f77e891ae 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVHeaderChecker.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVHeaderChecker.scala @@ -123,7 +123,7 @@ class CSVHeaderChecker( // Note: if there are only comments in the first block, the header would probably // be not extracted. if (options.headerFlag && isStartOfFile) { - CSVExpressionUtils.extractHeader(lines, options).foreach { header => + CSVExprUtils.extractHeader(lines, options).foreach { header => checkHeaderColumnNames(tokenizer.parseLine(header)) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala index 957238ff881d..3e25d820e994 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala @@ -83,7 +83,7 @@ class CSVOptions( } } - val delimiter = CSVExpressionUtils.toChar( + val delimiter = CSVExprUtils.toChar( parameters.getOrElse("sep", parameters.getOrElse("delimiter", ","))) val parseMode: ParseMode = parameters.get("mode").map(ParseMode.fromString).getOrElse(PermissiveMode) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala index e61771ae714e..46ed58ed9283 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala @@ -338,7 +338,7 @@ private[sql] object UnivocityParser { val options = parser.options - val filteredLines: Iterator[String] = CSVExpressionUtils.filterCommentAndEmpty(lines, options) + val filteredLines: Iterator[String] = CSVExprUtils.filterCommentAndEmpty(lines, options) val safeParser = new FailureSafeParser[String]( input => Seq(parser.parse(input)), diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVUtilsSuite.scala index 0d4f1f6eaa67..1133b3d3355d 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVUtilsSuite.scala @@ -21,40 +21,40 @@ import org.apache.spark.SparkFunSuite class CSVExpressionUtilsSuite extends SparkFunSuite { test("Can parse escaped characters") { - assert(CSVExpressionUtils.toChar("""\t""") === '\t') - assert(CSVExpressionUtils.toChar("""\r""") === '\r') - assert(CSVExpressionUtils.toChar("""\b""") === '\b') - assert(CSVExpressionUtils.toChar("""\f""") === '\f') - assert(CSVExpressionUtils.toChar("""\"""") === '\"') - assert(CSVExpressionUtils.toChar("""\'""") === '\'') - assert(CSVExpressionUtils.toChar("""\u0000""") === '\u0000') - assert(CSVExpressionUtils.toChar("""\\""") === '\\') + assert(CSVExprUtils.toChar("""\t""") === '\t') + assert(CSVExprUtils.toChar("""\r""") === '\r') + assert(CSVExprUtils.toChar("""\b""") === '\b') + assert(CSVExprUtils.toChar("""\f""") === '\f') + assert(CSVExprUtils.toChar("""\"""") === '\"') + assert(CSVExprUtils.toChar("""\'""") === '\'') + assert(CSVExprUtils.toChar("""\u0000""") === '\u0000') + assert(CSVExprUtils.toChar("""\\""") === '\\') } test("Does not accept delimiter larger than one character") { val exception = intercept[IllegalArgumentException]{ - CSVExpressionUtils.toChar("ab") + CSVExprUtils.toChar("ab") } assert(exception.getMessage.contains("cannot be more than one character")) } test("Throws exception for unsupported escaped characters") { val exception = intercept[IllegalArgumentException]{ - CSVExpressionUtils.toChar("""\1""") + CSVExprUtils.toChar("""\1""") } assert(exception.getMessage.contains("Unsupported special character for delimiter")) } test("string with one backward slash is prohibited") { val exception = intercept[IllegalArgumentException]{ - CSVExpressionUtils.toChar("""\""") + CSVExprUtils.toChar("""\""") } assert(exception.getMessage.contains("Single backslash is prohibited")) } test("output proper error message for empty string") { val exception = intercept[IllegalArgumentException]{ - CSVExpressionUtils.toChar("") + CSVExprUtils.toChar("") } assert(exception.getMessage.contains("Delimiter cannot be empty string")) } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala index 8b6945968181..21fabac472f4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/csv/CSVUtils.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.datasources.csv import org.apache.spark.rdd.RDD import org.apache.spark.sql.Dataset -import org.apache.spark.sql.catalyst.csv.CSVExpressionUtils +import org.apache.spark.sql.catalyst.csv.CSVExprUtils import org.apache.spark.sql.catalyst.csv.CSVOptions import org.apache.spark.sql.functions._ @@ -128,5 +128,5 @@ object CSVUtils { } def filterCommentAndEmpty(iter: Iterator[String], options: CSVOptions): Iterator[String] = - CSVExpressionUtils.filterCommentAndEmpty(iter, options) + CSVExprUtils.filterCommentAndEmpty(iter, options) }