diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/CrossDbmsQueryTestSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/CrossDbmsQueryTestSuite.scala index a693920c10032..4e24ff2f3489d 100644 --- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/CrossDbmsQueryTestSuite.scala +++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/querytest/CrossDbmsQueryTestSuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.jdbc import java.io.File +import java.nio.file.Files import java.sql.ResultSet import scala.collection.mutable.ArrayBuffer @@ -24,7 +25,6 @@ import scala.util.control.NonFatal import org.apache.spark.sql.Row import org.apache.spark.sql.SQLQueryTestHelper -import org.apache.spark.sql.catalyst.util.fileToString /** * This suite builds off of that to allow us to run other DBMS against the SQL test golden files (on @@ -76,7 +76,7 @@ trait CrossDbmsQueryTestSuite extends DockerJDBCIntegrationSuite with SQLQueryTe } protected def runSqlTestCase(testCase: TestCase, listTestCases: Seq[TestCase]): Unit = { - val input = fileToString(new File(testCase.inputFile)) + val input = Files.readString(new File(testCase.inputFile).toPath) val (comments, code) = splitCommentsAndCodes(input) val queries = getQueries(code, comments, listTestCases) @@ -143,7 +143,7 @@ trait CrossDbmsQueryTestSuite extends DockerJDBCIntegrationSuite with SQLQueryTe // Read back the golden files. var curSegment = 0 val expectedOutputs: Seq[QueryTestOutput] = { - val goldenOutput = fileToString(new File(testCase.resultFile)) + val goldenOutput = Files.readString(new File(testCase.resultFile).toPath) val segments = goldenOutput.split("-- !query.*\n") outputs.map { output => val result = diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala index 34528ebd39c34..1532f0e67b4db 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.catalyst import java.io._ -import java.nio.charset.Charset import java.nio.charset.StandardCharsets.UTF_8 import com.google.common.io.ByteStreams @@ -48,15 +47,6 @@ package object util extends Logging { } } - def fileToString(file: File, encoding: Charset = UTF_8): String = { - val inStream = new FileInputStream(file) - try { - new String(ByteStreams.toByteArray(inStream), encoding) - } finally { - inStream.close() - } - } - def resourceToBytes( resource: String, classLoader: ClassLoader = Utils.getSparkClassLoader): Array[Byte] = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala index 9977dcd83d6af..5a2eefcb0e932 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/SQLKeywordSuite.scala @@ -25,7 +25,6 @@ import scala.jdk.CollectionConverters._ import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.plans.SQLHelper -import org.apache.spark.sql.catalyst.util.fileToString trait SQLKeywordUtils extends SparkFunSuite with SQLHelper { @@ -38,7 +37,8 @@ trait SQLKeywordUtils extends SparkFunSuite with SQLHelper { getWorkspaceFilePath("sql", "api", "src", "main", "antlr4", "org", "apache", "spark", "sql", "catalyst", "parser", "SqlBaseLexer.g4").toFile - (fileToString(sqlBaseParserPath) + fileToString(sqlBaseLexerPath)).split("\n") + (Files.readString(sqlBaseParserPath.toPath) + + Files.readString(sqlBaseLexerPath.toPath)).split("\n") } // each element is an array of 4 string: the keyword name, reserve or not in Spark ANSI mode, @@ -47,7 +47,7 @@ trait SQLKeywordUtils extends SparkFunSuite with SQLHelper { val docPath = { getWorkspaceFilePath("docs", "sql-ref-ansi-compliance.md").toFile } - fileToString(docPath).split("\n") + Files.readString(docPath.toPath).split("\n") .dropWhile(!_.startsWith("|Keyword|")).drop(2).takeWhile(_.startsWith("|")) .map(_.stripPrefix("|").split("\\|").map(_.trim)) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala index f689c88a1cac2..fe222f8e102d4 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ExpressionsSchemaSuite.scala @@ -18,10 +18,11 @@ package org.apache.spark.sql import java.io.File +import java.nio.file.Files import scala.collection.mutable.ArrayBuffer -import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile} +import org.apache.spark.sql.catalyst.util.stringToFile import org.apache.spark.sql.test.SharedSparkSession import org.apache.spark.tags.ExtendedSQLTest import org.apache.spark.util.Utils @@ -165,7 +166,7 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession { val outputSize = outputs.size val headerSize = header.size val expectedOutputs = { - val expectedGoldenOutput = fileToString(resultFile) + val expectedGoldenOutput = Files.readString(resultFile.toPath) val lines = expectedGoldenOutput.split("\n") val expectedSize = lines.size diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ICUCollationsMapSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ICUCollationsMapSuite.scala index c9f0b9e54285c..be3f46f0b1c53 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ICUCollationsMapSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ICUCollationsMapSuite.scala @@ -17,8 +17,10 @@ package org.apache.spark.sql +import java.nio.file.Files + import org.apache.spark.SparkFunSuite -import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile, CollationFactory} +import org.apache.spark.sql.catalyst.util.{stringToFile, CollationFactory} import org.apache.spark.util.Utils // scalastyle:off line.size.limit @@ -61,7 +63,7 @@ class ICUCollationsMapSuite extends SparkFunSuite { } test("ICU locales map breaking change") { - val goldenLines = fileToString(collationsMapFile).split('\n') + val goldenLines = Files.readString(collationsMapFile.toPath).split('\n') val goldenRelevantLines = goldenLines.slice(4, goldenLines.length) // skip header val input = goldenRelevantLines.map( s => (s.split('|')(2).strip(), s.split('|')(1).strip().toInt)) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestHelper.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestHelper.scala index 9f728adcfb770..548f34cfdb77f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestHelper.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestHelper.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql import java.io.File +import java.nio.file.Files import scala.collection.mutable.ArrayBuffer import scala.util.control.NonFatal @@ -31,7 +32,6 @@ import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.catalyst.expressions.{CurrentDate, CurrentTime, CurrentTimestampLike, CurrentUser, Literal} import org.apache.spark.sql.catalyst.planning.PhysicalOperation import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.util.fileToString import org.apache.spark.sql.execution.HiveResult.hiveResultString import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.execution.command.{DescribeColumnCommand, DescribeCommandBase} @@ -410,7 +410,7 @@ trait SQLQueryTestHelper extends SQLConfHelper with Logging { val importedTestCaseName = comments.filter(_.startsWith("--IMPORT ")).map(_.substring(9)) val importedCode = importedTestCaseName.flatMap { testCaseName => allTestCases.find(_.name == testCaseName).map { testCase => - val input = fileToString(new File(testCase.inputFile)) + val input = Files.readString(new File(testCase.inputFile).toPath) val (_, code) = splitCommentsAndCodes(input) code } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index 6a7dd9c7c8929..a57c72f5fc155 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql import java.io.File import java.net.URI +import java.nio.file.Files import java.util.Locale import org.apache.spark.{SparkConf, TestUtils} @@ -27,8 +28,8 @@ import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.catalyst.plans.logical.{Command, LogicalPlan} import org.apache.spark.sql.catalyst.rules.RuleExecutor -import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile} import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_SECOND +import org.apache.spark.sql.catalyst.util.stringToFile import org.apache.spark.sql.execution.WholeStageCodegenExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.TimestampTypes @@ -229,7 +230,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper /** Run a test case. */ protected def runSqlTestCase(testCase: TestCase, listTestCases: Seq[TestCase]): Unit = { - val input = fileToString(new File(testCase.inputFile)) + val input = Files.readString(new File(testCase.inputFile).toPath) val (comments, code) = splitCommentsAndCodes(input) val queries = getQueries(code, comments, listTestCases) val settings = getSparkSettings(comments) @@ -639,7 +640,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper makeOutput: (String, Option[String], String) => QueryTestOutput): Unit = { // Read back the golden file. val expectedOutputs: Seq[QueryTestOutput] = { - val goldenOutput = fileToString(new File(resultFile)) + val goldenOutput = Files.readString(new File(resultFile).toPath) val segments = goldenOutput.split("-- !query.*\n") val numSegments = outputs.map(_.numSegments).sum + 1 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index d83fed4bf9a7d..7d1f060b973c7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -23,7 +23,7 @@ import java.nio.file.{Files, Paths} import scala.jdk.CollectionConverters._ import org.apache.spark.{SparkConf, SparkContext} -import org.apache.spark.sql.catalyst.util.{fileToString, resourceToString, stringToFile} +import org.apache.spark.sql.catalyst.util.{resourceToString, stringToFile} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.TestSparkSession import org.apache.spark.tags.ExtendedSQLTest @@ -130,7 +130,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp // Read back the golden file. val (expectedSchema, expectedOutput) = { - val goldenOutput = fileToString(goldenFile) + val goldenOutput = Files.readString(goldenFile.toPath) val segments = goldenOutput.split("-- !query.*\n") // query has 3 segments, plus the header diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala index 9f8b3b1686a94..e711be932ff3c 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.hive.thriftserver import java.io.File +import java.nio.file.Files import java.sql.{SQLException, Statement, Timestamp} import java.util.{Locale, MissingFormatArgumentException} @@ -27,7 +28,6 @@ import org.apache.spark.SparkException import org.apache.spark.internal.Logging import org.apache.spark.sql.SQLQueryTestSuite import org.apache.spark.sql.catalyst.analysis.NoSuchTableException -import org.apache.spark.sql.catalyst.util.fileToString import org.apache.spark.sql.execution.HiveResult.{getBinaryFormatter, getTimeFormatters, toHiveString, BinaryFormatter, TimeFormatters} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf.TimestampTypes @@ -151,7 +151,7 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServ // Read back the golden file. val expectedOutputs: Seq[QueryTestOutput] = { - val goldenOutput = fileToString(new File(testCase.resultFile)) + val goldenOutput = Files.readString(new File(testCase.resultFile).toPath) val segments = goldenOutput.split("-- !query.*\n") // each query has 3 segments, plus the header diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala index c4ccb07cb10b1..f7dcd28c50378 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.hive.execution import java.io._ import java.nio.charset.StandardCharsets +import java.nio.file.Files import java.util import java.util.Locale @@ -322,7 +323,7 @@ abstract class HiveComparisonTest extends SparkFunSuite with BeforeAndAfterAll { val hiveCachedResults = hiveCacheFiles.flatMap { cachedAnswerFile => logDebug(s"Looking for cached answer file $cachedAnswerFile.") if (cachedAnswerFile.exists) { - Some(fileToString(cachedAnswerFile)) + Some(Files.readString(cachedAnswerFile.toPath)) } else { logDebug(s"File $cachedAnswerFile not found") None diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala index 192fff2b98879..d515c9913cb17 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQueryFileTest.scala @@ -18,8 +18,7 @@ package org.apache.spark.sql.hive.execution import java.io.File - -import org.apache.spark.sql.catalyst.util._ +import java.nio.file.Files /** * A framework for running the query tests that are listed as a set of text files. @@ -67,7 +66,7 @@ abstract class HiveQueryFileTest extends HiveComparisonTest { realIncludeList.map(_.r.pattern.matcher(testCaseName).matches()).reduceLeft(_||_) || runAll) { // Build a test case and submit it to scala test framework... - val queriesString = fileToString(testCaseFile) + val queriesString = Files.readString(testCaseFile.toPath) createQueryTest(testCaseName, queriesString, reset = true, tryWithoutResettingFirst = true) } else { // Only output warnings for the built in includeList as this clutters the output when the