From 293be4e19b19e7a2b69ad7a0501596607182e868 Mon Sep 17 00:00:00 2001 From: Jiaan Geng Date: Sat, 10 Jun 2023 15:58:09 +0800 Subject: [PATCH 1/3] [SPARK-44018][SQL] Improve the hashCode for Some DS V2 Expression --- .../expressions/GeneralScalarExpression.java | 2 +- .../expressions/UserDefinedScalarFunc.java | 2 +- .../aggregate/GeneralAggregateFunc.java | 17 +++++++++++++++++ .../aggregate/UserDefinedAggregateFunc.java | 17 +++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/GeneralScalarExpression.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/GeneralScalarExpression.java index cb9bf6d69e2e..bd53fdf3382c 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/GeneralScalarExpression.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/GeneralScalarExpression.java @@ -447,6 +447,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(name, children); + return Objects.hash(name) * 31 + Arrays.hashCode(children); } } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/UserDefinedScalarFunc.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/UserDefinedScalarFunc.java index b7f603cd4316..98ee828aa69e 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/UserDefinedScalarFunc.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/UserDefinedScalarFunc.java @@ -58,6 +58,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(name, canonicalName, children); + return Objects.hash(name, canonicalName) * 31 + Arrays.hashCode(children); } } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/GeneralAggregateFunc.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/GeneralAggregateFunc.java index 1abf38656591..0427c441b245 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/GeneralAggregateFunc.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/GeneralAggregateFunc.java @@ -17,6 +17,9 @@ package org.apache.spark.sql.connector.expressions.aggregate; +import java.util.Arrays; +import java.util.Objects; + import org.apache.spark.annotation.Evolving; import org.apache.spark.sql.connector.expressions.Expression; import org.apache.spark.sql.internal.connector.ExpressionWithToString; @@ -60,4 +63,18 @@ public GeneralAggregateFunc(String name, boolean isDistinct, Expression[] childr @Override public Expression[] children() { return children; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GeneralAggregateFunc that = (GeneralAggregateFunc) o; + return Objects.equals(name, that.name) && Objects.equals(isDistinct, that.isDistinct) && + Arrays.equals(children, that.children); + } + + @Override + public int hashCode() { + return Objects.hash(name) * 31 + Arrays.hashCode(children); + } } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/UserDefinedAggregateFunc.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/UserDefinedAggregateFunc.java index d166ba16ba52..3e145750a6d8 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/UserDefinedAggregateFunc.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/UserDefinedAggregateFunc.java @@ -17,6 +17,9 @@ package org.apache.spark.sql.connector.expressions.aggregate; +import java.util.Arrays; +import java.util.Objects; + import org.apache.spark.annotation.Evolving; import org.apache.spark.sql.connector.expressions.Expression; import org.apache.spark.sql.internal.connector.ExpressionWithToString; @@ -50,4 +53,18 @@ public UserDefinedAggregateFunc( @Override public Expression[] children() { return children; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UserDefinedAggregateFunc that = (UserDefinedAggregateFunc) o; + return Objects.equals(name, that.name) && Objects.equals(canonicalName, that.canonicalName) && + Arrays.equals(children, that.children); + } + + @Override + public int hashCode() { + return Objects.hash(name, canonicalName) * 31 + Arrays.hashCode(children); + } } From 3b806f18aa022895582462fd6e4b8dcfaaf14da2 Mon Sep 17 00:00:00 2001 From: Jiaan Geng Date: Mon, 12 Jun 2023 09:27:41 +0800 Subject: [PATCH 2/3] Update code --- .../connector/expressions/aggregate/GeneralAggregateFunc.java | 4 ++-- .../expressions/aggregate/UserDefinedAggregateFunc.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/GeneralAggregateFunc.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/GeneralAggregateFunc.java index 0427c441b245..254f0a3936f9 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/GeneralAggregateFunc.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/GeneralAggregateFunc.java @@ -69,12 +69,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; GeneralAggregateFunc that = (GeneralAggregateFunc) o; - return Objects.equals(name, that.name) && Objects.equals(isDistinct, that.isDistinct) && + return Objects.equals(name, that.name) && isDistinct == that.isDistinct && Arrays.equals(children, that.children); } @Override public int hashCode() { - return Objects.hash(name) * 31 + Arrays.hashCode(children); + return Objects.hash(name, isDistinct) * 31 + Arrays.hashCode(children); } } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/UserDefinedAggregateFunc.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/UserDefinedAggregateFunc.java index 3e145750a6d8..7f2f06d5f602 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/UserDefinedAggregateFunc.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/UserDefinedAggregateFunc.java @@ -60,11 +60,11 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; UserDefinedAggregateFunc that = (UserDefinedAggregateFunc) o; return Objects.equals(name, that.name) && Objects.equals(canonicalName, that.canonicalName) && - Arrays.equals(children, that.children); + isDistinct == that.isDistinct && Arrays.equals(children, that.children); } @Override public int hashCode() { - return Objects.hash(name, canonicalName) * 31 + Arrays.hashCode(children); + return Objects.hash(name, canonicalName, isDistinct) * 31 + Arrays.hashCode(children); } } From 21589037a0c49570d42e70f6ac584fd838528b59 Mon Sep 17 00:00:00 2001 From: Jiaan Geng Date: Mon, 19 Jun 2023 11:30:04 +0800 Subject: [PATCH 3/3] Update code --- .../expressions/GeneralScalarExpression.java | 10 +++++++--- .../expressions/UserDefinedScalarFunc.java | 13 +++++++++---- .../aggregate/GeneralAggregateFunc.java | 13 +++++++++---- .../aggregate/UserDefinedAggregateFunc.java | 15 +++++++++++---- 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/GeneralScalarExpression.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/GeneralScalarExpression.java index bd53fdf3382c..859660600214 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/GeneralScalarExpression.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/GeneralScalarExpression.java @@ -18,7 +18,6 @@ package org.apache.spark.sql.connector.expressions; import java.util.Arrays; -import java.util.Objects; import org.apache.spark.annotation.Evolving; import org.apache.spark.sql.connector.expressions.filter.Predicate; @@ -441,12 +440,17 @@ public GeneralScalarExpression(String name, Expression[] children) { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; + GeneralScalarExpression that = (GeneralScalarExpression) o; - return Objects.equals(name, that.name) && Arrays.equals(children, that.children); + + if (!name.equals(that.name)) return false; + return Arrays.equals(children, that.children); } @Override public int hashCode() { - return Objects.hash(name) * 31 + Arrays.hashCode(children); + int result = name.hashCode(); + result = 31 * result + Arrays.hashCode(children); + return result; } } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/UserDefinedScalarFunc.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/UserDefinedScalarFunc.java index 98ee828aa69e..cbf3941d77d6 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/UserDefinedScalarFunc.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/UserDefinedScalarFunc.java @@ -18,7 +18,6 @@ package org.apache.spark.sql.connector.expressions; import java.util.Arrays; -import java.util.Objects; import org.apache.spark.annotation.Evolving; import org.apache.spark.sql.internal.connector.ExpressionWithToString; @@ -51,13 +50,19 @@ public UserDefinedScalarFunc(String name, String canonicalName, Expression[] chi public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; + UserDefinedScalarFunc that = (UserDefinedScalarFunc) o; - return Objects.equals(name, that.name) && Objects.equals(canonicalName, that.canonicalName) && - Arrays.equals(children, that.children); + + if (!name.equals(that.name)) return false; + if (!canonicalName.equals(that.canonicalName)) return false; + return Arrays.equals(children, that.children); } @Override public int hashCode() { - return Objects.hash(name, canonicalName) * 31 + Arrays.hashCode(children); + int result = name.hashCode(); + result = 31 * result + canonicalName.hashCode(); + result = 31 * result + Arrays.hashCode(children); + return result; } } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/GeneralAggregateFunc.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/GeneralAggregateFunc.java index 254f0a3936f9..4ef5b7f97e92 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/GeneralAggregateFunc.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/GeneralAggregateFunc.java @@ -18,7 +18,6 @@ package org.apache.spark.sql.connector.expressions.aggregate; import java.util.Arrays; -import java.util.Objects; import org.apache.spark.annotation.Evolving; import org.apache.spark.sql.connector.expressions.Expression; @@ -68,13 +67,19 @@ public GeneralAggregateFunc(String name, boolean isDistinct, Expression[] childr public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; + GeneralAggregateFunc that = (GeneralAggregateFunc) o; - return Objects.equals(name, that.name) && isDistinct == that.isDistinct && - Arrays.equals(children, that.children); + + if (isDistinct != that.isDistinct) return false; + if (!name.equals(that.name)) return false; + return Arrays.equals(children, that.children); } @Override public int hashCode() { - return Objects.hash(name, isDistinct) * 31 + Arrays.hashCode(children); + int result = name.hashCode(); + result = 31 * result + (isDistinct ? 1 : 0); + result = 31 * result + Arrays.hashCode(children); + return result; } } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/UserDefinedAggregateFunc.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/UserDefinedAggregateFunc.java index 7f2f06d5f602..10a62d0478b6 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/UserDefinedAggregateFunc.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/aggregate/UserDefinedAggregateFunc.java @@ -18,7 +18,6 @@ package org.apache.spark.sql.connector.expressions.aggregate; import java.util.Arrays; -import java.util.Objects; import org.apache.spark.annotation.Evolving; import org.apache.spark.sql.connector.expressions.Expression; @@ -58,13 +57,21 @@ public UserDefinedAggregateFunc( public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; + UserDefinedAggregateFunc that = (UserDefinedAggregateFunc) o; - return Objects.equals(name, that.name) && Objects.equals(canonicalName, that.canonicalName) && - isDistinct == that.isDistinct && Arrays.equals(children, that.children); + + if (isDistinct != that.isDistinct) return false; + if (!name.equals(that.name)) return false; + if (!canonicalName.equals(that.canonicalName)) return false; + return Arrays.equals(children, that.children); } @Override public int hashCode() { - return Objects.hash(name, canonicalName, isDistinct) * 31 + Arrays.hashCode(children); + int result = name.hashCode(); + result = 31 * result + canonicalName.hashCode(); + result = 31 * result + (isDistinct ? 1 : 0); + result = 31 * result + Arrays.hashCode(children); + return result; } }