diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala index b4c6e19d0bc39..677bdf2733612 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala @@ -1369,7 +1369,8 @@ case class SubqueryAlias( override def metadataOutput: Seq[Attribute] = { val qualifierList = identifier.qualifier :+ alias - child.metadataOutput.map(_.withQualifier(qualifierList)) + val nonHiddenMetadataOutput = child.metadataOutput.filter(!_.supportsQualifiedStar) + nonHiddenMetadataOutput.map(_.withQualifier(qualifierList)) } override def maxRows: Option[Long] = child.maxRows diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala index 5286a70674e78..de900fffb34a3 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala @@ -554,4 +554,26 @@ class DataFrameJoinSuite extends QueryTest ) } } + + test("SPARK-39376: Hide duplicated columns in star expansion of subquery alias from USING JOIN") { + val joinDf = testData2.as("testData2").join( + testData3.as("testData3"), usingColumns = Seq("a"), joinType = "fullouter") + val equivalentQueries = Seq( + joinDf.select($"*"), + joinDf.as("r").select($"*"), + joinDf.as("r").select($"r.*") + ) + equivalentQueries.foreach { query => + checkAnswer(query, + Seq( + Row(1, 1, null), + Row(1, 2, null), + Row(2, 1, 2), + Row(2, 2, 2), + Row(3, 1, null), + Row(3, 2, null) + ) + ) + } + } }