From b59ed06fa4b6f97bd0608780a441eaccd1c064b7 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 6 Sep 2018 22:47:16 -0700 Subject: [PATCH] simplify --- .../execution/datasources/orc/OrcUtils.scala | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala index 5e13c66bb4104..c76f1fe29391b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala @@ -17,8 +17,6 @@ package org.apache.spark.sql.execution.datasources.orc -import java.util.Locale - import scala.collection.JavaConverters._ import org.apache.hadoop.conf.Configuration @@ -29,7 +27,7 @@ import org.apache.spark.SparkException import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.analysis.caseSensitiveResolution +import org.apache.spark.sql.catalyst.analysis.{caseInsensitiveResolution, caseSensitiveResolution} import org.apache.spark.sql.catalyst.parser.CatalystSqlParser import org.apache.spark.sql.types._ @@ -117,29 +115,16 @@ object OrcUtils extends Logging { } }) } else { - if (isCaseSensitive) { - Some(requiredSchema.fieldNames.map { name => - orcFieldNames.indexWhere(caseSensitiveResolution(_, name)) - }) - } else { - // Do case-insensitive resolution only if in case-insensitive mode - val caseInsensitiveOrcFieldMap = - orcFieldNames.zipWithIndex.groupBy(_._1.toLowerCase(Locale.ROOT)) - Some(requiredSchema.fieldNames.map { requiredFieldName => - caseInsensitiveOrcFieldMap - .get(requiredFieldName.toLowerCase(Locale.ROOT)) - .map { matchedOrcFields => - if (matchedOrcFields.size > 1) { - // Need to fail if there is ambiguity, i.e. more than one field is matched. - val matchedOrcFieldsString = matchedOrcFields.map(_._1).mkString("[", ", ", "]") - throw new RuntimeException(s"""Found duplicate field(s) "$requiredFieldName": """ - + s"$matchedOrcFieldsString in case-insensitive mode") - } else { - matchedOrcFields.head._2 - } - }.getOrElse(-1) - }) + val resolver = if (isCaseSensitive) caseSensitiveResolution else caseInsensitiveResolution + // Need to fail if there is ambiguity, i.e. more than one field is matched. + requiredSchema.fieldNames.foreach { requiredFieldName => + val matchedOrcFields = orcFieldNames.filter(resolver(_, requiredFieldName)) + if (matchedOrcFields.size > 1) { + throw new RuntimeException(s"""Found duplicate field(s) "$requiredFieldName": """ + + s"${matchedOrcFields.mkString("[", ", ", "]")} in case-insensitive mode") + } } + Some(requiredSchema.fieldNames.map { name => orcFieldNames.indexWhere(resolver(_, name)) }) } } }