Skip to content

Commit 8b5d900

Browse files
LuciferYangdongjoon-hyun
authored andcommitted
[SPARK-54578][SQL] Perform Code Cleanup on AssignmentUtils
### What changes were proposed in this pull request? This PR performs a code cleanup on `AssignmentUtils`: 1. Deleted the redundant private methods `getMissingSourcePaths` and `createNullCheckForFieldPath`, which were introduced in #53149 but became obsolete after #53229. 2. Removed the redundant documentation for the invalid input parameter `missingSourcePaths` in the `alignUpdateAssignments` method. 3. Removed the unused input parameter `updateStar` from the private method `applyAssignments`. 4. Corrected a spelling error in the input parameter `coerceNestedTypes` of the private method `applyFieldAssignments`. ### Why are the changes needed? Code Cleanup. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? - Pass Github Actions ### Was this patch authored or co-authored using generative AI tooling? No Closes #53302 from LuciferYang/SPARK-54578. Lead-authored-by: yangjie01 <[email protected]> Co-authored-by: YangJie <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent 7bea0b7 commit 8b5d900

File tree

1 file changed

+5
-54
lines changed

1 file changed

+5
-54
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala

Lines changed: 5 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import scala.collection.mutable
2121

2222
import org.apache.spark.sql.catalyst.SQLConfHelper
2323
import org.apache.spark.sql.catalyst.analysis.TableOutputResolver.DefaultValueFillMode.{NONE, RECURSE}
24-
import org.apache.spark.sql.catalyst.expressions.{Attribute, CreateNamedStruct, Expression, GetStructField, IsNull, Literal}
24+
import org.apache.spark.sql.catalyst.expressions.{Attribute, CreateNamedStruct, Expression, GetStructField, Literal}
2525
import org.apache.spark.sql.catalyst.plans.logical.Assignment
2626
import org.apache.spark.sql.catalyst.types.DataTypeUtils
2727
import org.apache.spark.sql.catalyst.util.CharVarcharUtils
@@ -55,7 +55,6 @@ object AssignmentUtils extends SQLConfHelper with CastSupport {
5555
* (preserving existing fields).
5656
* @param coerceNestedTypes whether to coerce nested types to match the target type
5757
* for complex types
58-
* @param missingSourcePaths paths that exist in target but not in source
5958
* @return aligned update assignments that match table attributes
6059
*/
6160
def alignUpdateAssignments(
@@ -73,8 +72,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport {
7372
assignments,
7473
addError = err => errors += err,
7574
colPath = Seq(attr.name),
76-
coerceNestedTypes,
77-
fromStar)
75+
coerceNestedTypes)
7876
}
7977

8078
if (errors.nonEmpty) {
@@ -158,8 +156,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport {
158156
assignments: Seq[Assignment],
159157
addError: String => Unit,
160158
colPath: Seq[String],
161-
coerceNestedTypes: Boolean = false,
162-
updateStar: Boolean = false): Expression = {
159+
coerceNestedTypes: Boolean = false): Expression = {
163160

164161
val (exactAssignments, otherAssignments) = assignments.partition { assignment =>
165162
assignment.key.semanticEquals(colExpr)
@@ -197,7 +194,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport {
197194
assignments: Seq[Assignment],
198195
addError: String => Unit,
199196
colPath: Seq[String],
200-
coerceNestedTyptes: Boolean): Expression = {
197+
coerceNestedTypes: Boolean): Expression = {
201198

202199
col.dataType match {
203200
case structType: StructType =>
@@ -207,7 +204,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport {
207204
}
208205
val updatedFieldExprs = fieldAttrs.zip(fieldExprs).map { case (fieldAttr, fieldExpr) =>
209206
applyAssignments(fieldAttr, fieldExpr, assignments, addError, colPath :+ fieldAttr.name,
210-
coerceNestedTyptes)
207+
coerceNestedTypes)
211208
}
212209
toNamedStruct(structType, updatedFieldExprs)
213210

@@ -226,52 +223,6 @@ object AssignmentUtils extends SQLConfHelper with CastSupport {
226223
CreateNamedStruct(namedStructExprs)
227224
}
228225

229-
private def getMissingSourcePaths(targetType: StructType,
230-
sourceType: DataType,
231-
colPath: Seq[String],
232-
addError: String => Unit): Seq[Seq[String]] = {
233-
val nestedTargetPaths = DataTypeUtils.extractLeafFieldPaths(targetType, Seq.empty)
234-
val nestedSourcePaths = sourceType match {
235-
case sourceStructType: StructType =>
236-
DataTypeUtils.extractLeafFieldPaths(sourceStructType, Seq.empty)
237-
case _ =>
238-
addError(s"Value for struct type: " +
239-
s"${colPath.quoted} must be a struct but was ${sourceType.simpleString}")
240-
Seq()
241-
}
242-
nestedSourcePaths.diff(nestedTargetPaths)
243-
}
244-
245-
/**
246-
* Creates a null check for a field at the given path within a struct expression.
247-
* Navigates through the struct hierarchy following the path and returns an IsNull check
248-
* for the final field.
249-
*
250-
* @param rootExpr the root expression to navigate from
251-
* @param path the field path to navigate (sequence of field names)
252-
* @return an IsNull expression checking if the field at the path is null
253-
*/
254-
private def createNullCheckForFieldPath(
255-
rootExpr: Expression,
256-
path: Seq[String]): Expression = {
257-
var currentExpr: Expression = rootExpr
258-
path.foreach { fieldName =>
259-
currentExpr.dataType match {
260-
case st: StructType =>
261-
st.fields.find(f => conf.resolver(f.name, fieldName)) match {
262-
case Some(field) =>
263-
val fieldIndex = st.fieldIndex(field.name)
264-
currentExpr = GetStructField(currentExpr, fieldIndex, Some(field.name))
265-
case None =>
266-
// Field not found, shouldn't happen
267-
}
268-
case _ =>
269-
// Not a struct, shouldn't happen
270-
}
271-
}
272-
IsNull(currentExpr)
273-
}
274-
275226
/**
276227
* Checks whether assignments are aligned and compatible with table columns.
277228
*

0 commit comments

Comments
 (0)