diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index d08a6382f738b..6e4b1b241ae4f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -200,6 +200,8 @@ class Analyzer( val postHocResolutionRules: Seq[Rule[LogicalPlan]] = Nil lazy val batches: Seq[Batch] = Seq( + Batch("Disable Hints", Once, + new ResolveHints.DisableHints(conf)), Batch("Hints", fixedPoint, new ResolveHints.ResolveJoinStrategyHints(conf), new ResolveHints.ResolveCoalesceHints(conf)), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala index 4cbff62e16cc1..120842b0c4a07 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveHints.scala @@ -278,4 +278,15 @@ object ResolveHints { h.child } } + + /** + * Removes all the hints when `spark.sql.optimizer.disableHints` is set. + * This is executed at the very beginning of the Analyzer to disable + * the hint functionality. + */ + class DisableHints(conf: SQLConf) extends RemoveAllHints(conf: SQLConf) { + override def apply(plan: LogicalPlan): LogicalPlan = { + if (conf.getConf(SQLConf.DISABLE_HINTS)) super.apply(plan) else plan + } + } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 3149d14c1ddcc..18b9bbe1bce00 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2106,6 +2106,15 @@ object SQLConf { .booleanConf .createWithDefault(true) + val DISABLE_HINTS = + buildConf("spark.sql.optimizer.disableHints") + .internal() + .doc("When true, the optimizer will disable user-specified hints that are additional " + + "directives for better planning of a query.") + .version("3.1.0") + .booleanConf + .createWithDefault(false) + val NESTED_PREDICATE_PUSHDOWN_FILE_SOURCE_LIST = buildConf("spark.sql.optimizer.nestedPredicatePushdown.supportedFileSources") .internal() diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index a219b91627b2b..989f304b1f07f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -3521,6 +3521,45 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark |""".stripMargin), Row(1)) } } + + test("SPARK-31875: remove hints from plan when spark.sql.optimizer.disableHints = true") { + withSQLConf(SQLConf.DISABLE_HINTS.key -> "true") { + withTempView("t1", "t2") { + Seq[Integer](1, 2).toDF("c1").createOrReplaceTempView("t1") + Seq[Integer](1, 2).toDF("c1").createOrReplaceTempView("t2") + val repartitionHints = Seq( + "COALESCE(2)", + "REPARTITION(c1)", + "REPARTITION(c1, 2)", + "REPARTITION_BY_RANGE(c1, 2)", + "REPARTITION_BY_RANGE(c1)" + ) + val joinHints = Seq( + "BROADCASTJOIN (t1)", + "MAPJOIN(t1)", + "SHUFFLE_MERGE(t1)", + "MERGEJOIN(t1)", + "SHUFFLE_REPLICATE_NL(t1)" + ) + + repartitionHints.foreach { hintName => + val sqlText = s"SELECT /*+ $hintName */ * FROM t1" + val sqlTextWithoutHint = "SELECT * FROM t1" + val expectedPlan = sql(sqlTextWithoutHint) + val actualPlan = sql(sqlText) + comparePlans(actualPlan.queryExecution.analyzed, expectedPlan.queryExecution.analyzed) + } + + joinHints.foreach { hintName => + val sqlText = s"SELECT /*+ $hintName */ * FROM t1 INNER JOIN t2 ON t1.c1 = t2.c1" + val sqlTextWithoutHint = "SELECT * FROM t1 INNER JOIN t2 ON t1.c1 = t2.c1" + val expectedPlan = sql(sqlTextWithoutHint) + val actualPlan = sql(sqlText) + comparePlans(actualPlan.queryExecution.analyzed, expectedPlan.queryExecution.analyzed) + } + } + } + } } case class Foo(bar: Option[String])