From a19fd385905593fff02d2afa86173e1a80bb638e Mon Sep 17 00:00:00 2001 From: tpp Date: Tue, 26 Aug 2025 01:11:56 -0700 Subject: [PATCH 01/18] planner: Add variable for no_decorrelate --- pkg/planner/core/expression_rewriter.go | 9 ++++++++- pkg/sessionctx/vardef/tidb_vars.go | 4 ++++ pkg/sessionctx/variable/session.go | 3 +++ pkg/sessionctx/variable/setvar_affect.go | 1 + pkg/sessionctx/variable/sysvar.go | 4 ++++ 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index 8fb3628718cfa..120333bb27770 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -1265,8 +1265,15 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp } func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64) bool { + // Check if NO_DECORRELATE hint is explicitly specified noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 - if noDecorrelate && len(corCols) == 0 { + + if len(corCols) > 0 { + // If no explicit hint, check if the session variable is enabled + if !noDecorrelate { + noDecorrelate = planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate + } + } else if noDecorrelate { planCtx.builder.ctx.GetSessionVars().StmtCtx.SetHintWarning( "NO_DECORRELATE() is inapplicable because there are no correlated columns.") noDecorrelate = false diff --git a/pkg/sessionctx/vardef/tidb_vars.go b/pkg/sessionctx/vardef/tidb_vars.go index 6b605f89c554c..6122e2ff40d05 100644 --- a/pkg/sessionctx/vardef/tidb_vars.go +++ b/pkg/sessionctx/vardef/tidb_vars.go @@ -598,6 +598,9 @@ const ( // TiDBEnablePipelinedWindowFunction is used to control whether to use pipelined window function, it only works when tidb_enable_window_function = true. TiDBEnablePipelinedWindowFunction = "tidb_enable_pipelined_window_function" + // TiDBEnableNoDecorrelate is used to control whether to enable the NO_DECORRELATE hint globally or at session level. + TiDBEnableNoDecorrelate = "tidb_enable_no_decorrelate" + // TiDBEnableStrictDoubleTypeCheck is used to control table field double type syntax check. TiDBEnableStrictDoubleTypeCheck = "tidb_enable_strict_double_type_check" @@ -1437,6 +1440,7 @@ const ( DefTiDBForcePriority = mysql.NoPriority DefEnableWindowFunction = true DefEnablePipelinedWindowFunction = true + DefEnableNoDecorrelate = false DefEnableStrictDoubleTypeCheck = true DefEnableVectorizedExpression = true DefTiDBOptJoinReorderThreshold = 0 diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index b47e78f76a76b..62e40874d8adb 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -1133,6 +1133,9 @@ type SessionVars struct { // EnablePipelinedWindowExec enables executing window functions in a pipelined manner. EnablePipelinedWindowExec bool + // EnableNoDecorrelate enables the NO_DECORRELATE hint globally or at session level. + EnableNoDecorrelate bool + // AllowProjectionPushDown enables pushdown projection on TiKV. AllowProjectionPushDown bool diff --git a/pkg/sessionctx/variable/setvar_affect.go b/pkg/sessionctx/variable/setvar_affect.go index 24d2b066919c8..22e74fd49380e 100644 --- a/pkg/sessionctx/variable/setvar_affect.go +++ b/pkg/sessionctx/variable/setvar_affect.go @@ -129,6 +129,7 @@ var isHintUpdatableVerified = map[string]struct{}{ "tiflash_fine_grained_shuffle_batch_size": {}, "tiflash_fine_grained_shuffle_stream_count": {}, "tidb_hash_join_version": {}, + "tidb_enable_no_decorrelate": {}, // Variables that is compatible with MySQL. "cte_max_recursion_depth": {}, "sql_mode": {}, diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index 492c7a96ca1e1..e0f4344d9d27c 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -2356,6 +2356,10 @@ var defaultSysVars = []*SysVar{ s.EnablePipelinedWindowExec = TiDBOptOn(val) return nil }}, + {Scope: vardef.ScopeGlobal | vardef.ScopeSession, Name: vardef.TiDBEnableNoDecorrelate, Value: BoolToOnOff(vardef.DefEnableNoDecorrelate), Type: vardef.TypeBool, SetSession: func(s *SessionVars, val string) error { + s.EnableNoDecorrelate = TiDBOptOn(val) + return nil + }}, {Scope: vardef.ScopeGlobal | vardef.ScopeSession, Name: vardef.TiDBEnableStrictDoubleTypeCheck, Value: BoolToOnOff(vardef.DefEnableStrictDoubleTypeCheck), Type: vardef.TypeBool, SetSession: func(s *SessionVars, val string) error { s.EnableStrictDoubleTypeCheck = TiDBOptOn(val) return nil From dcbf88899e44a8dbaf402a98d327db49a0ce0eef Mon Sep 17 00:00:00 2001 From: tpp Date: Wed, 27 Aug 2025 01:17:35 -0700 Subject: [PATCH 02/18] restrict to scalar only --- pkg/planner/core/expression_rewriter.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index 120333bb27770..ea4b5a05e4e60 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -1265,15 +1265,9 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp } func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64) bool { - // Check if NO_DECORRELATE hint is explicitly specified noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 - if len(corCols) > 0 { - // If no explicit hint, check if the session variable is enabled - if !noDecorrelate { - noDecorrelate = planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate - } - } else if noDecorrelate { + if noDecorrelate && len(corCols) == 0 { planCtx.builder.ctx.GetSessionVars().StmtCtx.SetHintWarning( "NO_DECORRELATE() is inapplicable because there are no correlated columns.") noDecorrelate = false @@ -1293,6 +1287,10 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, planCtx np = planCtx.builder.buildMaxOneRow(np) correlatedColumn := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags) + if !noDecorrelate && len(correlatedColumn) > 0 { + // If no explicit hint, check if the session variable is enabled + noDecorrelate = planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate + } if planCtx.builder.disableSubQueryPreprocessing || len(coreusage.ExtractCorrelatedCols4LogicalPlan(np)) > 0 || hasCTEConsumerInSubPlan(np) { planCtx.plan = planCtx.builder.buildApplyWithJoinType(planCtx.plan, np, logicalop.LeftOuterJoin, noDecorrelate) From c3241124dc97019386371e83bb2556495ec58bd1 Mon Sep 17 00:00:00 2001 From: tpp Date: Wed, 27 Aug 2025 02:20:45 -0700 Subject: [PATCH 03/18] comprehensive refactor --- pkg/planner/core/expression_rewriter.go | 105 +++++++++++++++++++++-- pkg/planner/core/logical_plan_builder.go | 35 ++++---- pkg/planner/core/planbuilder.go | 5 ++ 3 files changed, 117 insertions(+), 28 deletions(-) diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index ea4b5a05e4e60..26014c32375d7 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -735,7 +735,7 @@ func (er *expressionRewriter) handleCompareSubquery(ctx context.Context, planCtx return v, true } corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, np) // Only (a,b,c) = any (...) and (a,b,c) != all (...) can use row expression. canMultiCol := (!v.All && v.Op == opcode.EQ) || (v.All && v.Op == opcode.NE) @@ -1041,7 +1041,8 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, planCtx * } np = er.popExistsSubPlan(planCtx, np) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, np) + semiJoinRewrite := hintFlags&hint.HintFlagSemiJoinRewrite > 0 if semiJoinRewrite && noDecorrelate { b.ctx.GetSessionVars().StmtCtx.SetHintWarning( @@ -1212,7 +1213,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp lt, rt := lexpr.GetType(er.sctx.GetEvalCtx()), rexpr.GetType(er.sctx.GetEvalCtx()) collFlag := collate.CompatibleCollate(lt.GetCollate(), rt.GetCollate()) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, np) // If it's not the form of `not in (SUBQUERY)`, // and has no correlated column from the current level plan(if the correlated column is from upper level, @@ -1264,7 +1265,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp return v, true } -func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64) bool { +func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64, subqueryPlan base.LogicalPlan) bool { noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 if noDecorrelate && len(corCols) == 0 { @@ -1272,9 +1273,99 @@ func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.Correla "NO_DECORRELATE() is inapplicable because there are no correlated columns.") noDecorrelate = false } + + // If decorrelation is allowed, check if correlation columns are index-covered + if !noDecorrelate && len(corCols) > 0 { + // Check session variable + noDecorrelate = planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate + + // If still allowed, check index coverage + if !noDecorrelate && subqueryPlan != nil { + tableInfo := getTableInfoFromPlan(subqueryPlan) + if tableInfo != nil && !canEfficientlyEvaluateCorrelation(corCols, tableInfo) { + // Disable decorrelation if correlation columns are not index-covered + noDecorrelate = true + } + } + } + return noDecorrelate } +// getTableInfoFromPlan extracts table info from a logical plan +func getTableInfoFromPlan(p base.LogicalPlan) *model.TableInfo { + // Try to find table info by traversing the plan tree + switch v := p.(type) { + case *logicalop.DataSource: + return v.TableInfo + case *logicalop.LogicalProjection: + if len(v.Children()) > 0 { + return getTableInfoFromPlan(v.Children()[0]) + } + case *logicalop.LogicalSelection: + if len(v.Children()) > 0 { + return getTableInfoFromPlan(v.Children()[0]) + } + case *logicalop.LogicalAggregation: + if len(v.Children()) > 0 { + return getTableInfoFromPlan(v.Children()[0]) + } + case *logicalop.LogicalSort: + if len(v.Children()) > 0 { + return getTableInfoFromPlan(v.Children()[0]) + } + case *logicalop.LogicalLimit: + if len(v.Children()) > 0 { + return getTableInfoFromPlan(v.Children()[0]) + } + } + return nil +} + +// canEfficientlyEvaluateCorrelation checks if correlation columns are covered by any index +func canEfficientlyEvaluateCorrelation(corCols []*expression.CorrelatedColumn, tableInfo *model.TableInfo) bool { + if len(corCols) == 0 || tableInfo == nil { + return false + } + + // Extract column IDs from correlation columns + corColIDs := make(map[int64]bool) + for _, corCol := range corCols { + corColIDs[corCol.Column.UniqueID] = true + } + + // Check if any index covers these correlation columns + for _, idx := range tableInfo.Indices { + if idx.State != model.StatePublic { + continue + } + + // Check if index prefix covers correlation columns + for i, col := range idx.Columns { + // Get the actual column ID from the table info using the offset + if col.Offset < len(tableInfo.Columns) { + colID := tableInfo.Columns[col.Offset].ID + if corColIDs[colID] { + // Found a correlation column in index, check if it's in prefix + if i == 0 || (i > 0 && idx.Columns[i-1].Offset != col.Offset) { + return true + } + } + } + } + } + + // Check primary key coverage + if tableInfo.PKIsHandle { + pkCol := tableInfo.GetPkColInfo() + if pkCol != nil && corColIDs[pkCol.ID] { + return true + } + } + + return false +} + func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, planCtx *exprRewriterPlanCtx, v *ast.SubqueryExpr) (ast.Node, bool) { intest.AssertNotNil(planCtx) ci := planCtx.builder.prepareCTECheckForSubQuery() @@ -1286,11 +1377,7 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, planCtx } np = planCtx.builder.buildMaxOneRow(np) correlatedColumn := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags) - if !noDecorrelate && len(correlatedColumn) > 0 { - // If no explicit hint, check if the session variable is enabled - noDecorrelate = planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate - } + noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags, np) if planCtx.builder.disableSubQueryPreprocessing || len(coreusage.ExtractCorrelatedCols4LogicalPlan(np)) > 0 || hasCTEConsumerInSubPlan(np) { planCtx.plan = planCtx.builder.buildApplyWithJoinType(planCtx.plan, np, logicalop.LeftOuterJoin, noDecorrelate) diff --git a/pkg/planner/core/logical_plan_builder.go b/pkg/planner/core/logical_plan_builder.go index d15f9a01a048f..f72198d87e671 100644 --- a/pkg/planner/core/logical_plan_builder.go +++ b/pkg/planner/core/logical_plan_builder.go @@ -1992,7 +1992,7 @@ func (b *PlanBuilder) checkOrderByInDistinct(byItem *ast.ByItem, idx int, expr e // Check if referenced columns of expressions in ORDER BY whole match some fields in DISTINCT, // both original expression and alias can be referenced. // e.g. - // select distinct a from t order by sin(a); ✔ + // select distinct sin(a) from t order by a; ✔ // select distinct a, b from t order by a+b; ✔ // select distinct count(a), sum(a) from t group by b order by sum(a); ✔ cols := expression.ExtractColumns(expr) @@ -2546,6 +2546,10 @@ func (b *PlanBuilder) extractCorrelatedAggFuncs(ctx context.Context, p base.Logi corCols = append(corCols, expression.ExtractCorColumns(expr)...) cols = append(cols, expression.ExtractColumns(expr)...) } + // If decorrelation is disabled, don't extract correlated aggregates + if b.noDecorrelate && len(corCols) > 0 { + continue + } if len(corCols) > 0 && len(cols) == 0 { outer = append(outer, agg) } @@ -2610,6 +2614,9 @@ type correlatedAggregateResolver struct { // correlatedAggFuncs stores aggregate functions which belong to outer query correlatedAggFuncs []*ast.AggregateFuncExpr + + // noDecorrelate indicates whether decorrelation should be disabled for this resolver + noDecorrelate bool } // Enter implements Visitor interface. @@ -2746,7 +2753,7 @@ func (r *correlatedAggregateResolver) collectFromGroupBy(p base.LogicalPlan, gro r.b.curClause = groupByClause outerAggFuncs, err := r.b.extractCorrelatedAggFuncs(r.ctx, p, aggList) if err != nil { - return nil + return err } r.correlatedAggFuncs = append(r.correlatedAggFuncs, outerAggFuncs...) return nil @@ -2784,9 +2791,10 @@ func (r *correlatedAggregateResolver) Leave(n ast.Node) (ast.Node, bool) { // in the outer query from all the sub-queries inside SELECT fields. func (b *PlanBuilder) resolveCorrelatedAggregates(ctx context.Context, sel *ast.SelectStmt, p base.LogicalPlan) (map[*ast.AggregateFuncExpr]int, error) { resolver := &correlatedAggregateResolver{ - ctx: ctx, - b: b, - outerPlan: p, + ctx: ctx, + b: b, + outerPlan: p, + noDecorrelate: b.noDecorrelate, } correlatedAggList := make([]*ast.AggregateFuncExpr, 0) for _, field := range sel.Fields.Fields { @@ -5823,11 +5831,11 @@ func (b *PlanBuilder) buildUpdateLists(ctx context.Context, tableList []*ast.Tab newList = make([]*expression.Assignment, 0, p.Schema().Len()) tblDbMap := make(map[string]string, len(tableList)) for _, tbl := range tableList { - tblW := b.resolveCtx.GetTableName(tbl) - if isCTE(tblW) { + tnW := b.resolveCtx.GetTableName(tbl) + if isCTE(tnW) { continue } - tblDbMap[tbl.Name.L] = tblW.DBInfo.Name.L + tblDbMap[tbl.Name.L] = tnW.DBInfo.Name.L } allAssignments := append(list, virtualAssignments...) @@ -6373,17 +6381,6 @@ func (b *PlanBuilder) buildWindowFunctionFrameBound(_ context.Context, spec *ast return bound, nil } - bound.CalcFuncs = make([]expression.Expression, len(orderByItems)) - bound.CmpFuncs = make([]expression.CompareFunc, len(orderByItems)) - if bound.Type == ast.CurrentRow { - for i, item := range orderByItems { - col := item.Col - bound.CalcFuncs[i] = col - bound.CmpFuncs[i] = expression.GetCmpFunction(b.ctx.GetExprCtx(), col, col) - } - return bound, nil - } - col := orderByItems[0].Col // TODO: We also need to raise error for non-deterministic expressions, like rand(). val, err := evalAstExprWithPlanCtx(b.ctx, boundClause.Expr) diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index 5cfb0f585e463..930c7b8f026c7 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -303,6 +303,9 @@ type PlanBuilder struct { // disableSubQueryPreprocessing indicates whether to pre-process uncorrelated sub-queries in rewriting stage. disableSubQueryPreprocessing bool + // noDecorrelate indicates whether decorrelation should be disabled for correlated aggregates in subqueries + noDecorrelate bool + // allowBuildCastArray indicates whether allow cast(... as ... array). allowBuildCastArray bool // resolveCtx is set when calling Build, it's only effective in the current Build call. @@ -462,6 +465,7 @@ func (b *PlanBuilder) Init(sctx base.PlanContext, is infoschema.InfoSchema, proc b.is = is b.hintProcessor = processor b.isForUpdateRead = sctx.GetSessionVars().IsPessimisticReadConsistency() + b.noDecorrelate = sctx.GetSessionVars().EnableNoDecorrelate if savedBlockNames == nil { return b, nil } @@ -494,6 +498,7 @@ func (b *PlanBuilder) ResetForReuse() *PlanBuilder { b.colMapper = saveColMapper b.handleHelper = saveHandleHelper b.correlatedAggMapper = saveCorrelateAggMapper + b.noDecorrelate = false // Add more fields if they are safe to be reused. From 8cd4fdfa35b4ba06886048cdfca8af8be5e03aa7 Mon Sep 17 00:00:00 2001 From: tpp Date: Wed, 27 Aug 2025 15:33:24 -0700 Subject: [PATCH 04/18] comprehensive refactor2 --- pkg/planner/core/expression_rewriter.go | 47 +++++++------------------ 1 file changed, 12 insertions(+), 35 deletions(-) diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index 26014c32375d7..fba625d0d88c2 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -1268,24 +1268,17 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64, subqueryPlan base.LogicalPlan) bool { noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 - if noDecorrelate && len(corCols) == 0 { - planCtx.builder.ctx.GetSessionVars().StmtCtx.SetHintWarning( - "NO_DECORRELATE() is inapplicable because there are no correlated columns.") - noDecorrelate = false - } - - // If decorrelation is allowed, check if correlation columns are index-covered - if !noDecorrelate && len(corCols) > 0 { - // Check session variable - noDecorrelate = planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate - - // If still allowed, check index coverage - if !noDecorrelate && subqueryPlan != nil { - tableInfo := getTableInfoFromPlan(subqueryPlan) - if tableInfo != nil && !canEfficientlyEvaluateCorrelation(corCols, tableInfo) { - // Disable decorrelation if correlation columns are not index-covered - noDecorrelate = true - } + if noDecorrelate { + if len(corCols) == 0 { + planCtx.builder.ctx.GetSessionVars().StmtCtx.SetHintWarning( + "NO_DECORRELATE() is inapplicable because there are no correlated columns.") + noDecorrelate = false + } + } else if len(corCols) > 0 && planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate && subqueryPlan != nil { + tableInfo := getTableInfoFromPlan(subqueryPlan) + if tableInfo != nil && canEfficientlyEvaluateCorrelation(corCols, tableInfo) { + // Only enable decorrelation by the session variable if correlation columns are index-covered + noDecorrelate = true } } @@ -1298,23 +1291,7 @@ func getTableInfoFromPlan(p base.LogicalPlan) *model.TableInfo { switch v := p.(type) { case *logicalop.DataSource: return v.TableInfo - case *logicalop.LogicalProjection: - if len(v.Children()) > 0 { - return getTableInfoFromPlan(v.Children()[0]) - } - case *logicalop.LogicalSelection: - if len(v.Children()) > 0 { - return getTableInfoFromPlan(v.Children()[0]) - } - case *logicalop.LogicalAggregation: - if len(v.Children()) > 0 { - return getTableInfoFromPlan(v.Children()[0]) - } - case *logicalop.LogicalSort: - if len(v.Children()) > 0 { - return getTableInfoFromPlan(v.Children()[0]) - } - case *logicalop.LogicalLimit: + case *logicalop.LogicalProjection, *logicalop.LogicalSelection, *logicalop.LogicalAggregation, *logicalop.LogicalSort, *logicalop.LogicalLimit, *logicalop.LogicalMaxOneRow: if len(v.Children()) > 0 { return getTableInfoFromPlan(v.Children()[0]) } From 154fc6785bc3f73a9836993518abf48ba1c82ad8 Mon Sep 17 00:00:00 2001 From: tpp Date: Wed, 27 Aug 2025 16:36:14 -0700 Subject: [PATCH 05/18] comprehensive refactor3 --- pkg/planner/core/expression_rewriter.go | 13 +++++++------ pkg/planner/core/logical_plan_builder.go | 19 +++++++++++++++---- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index fba625d0d88c2..7fc1237d9e66a 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -735,7 +735,7 @@ func (er *expressionRewriter) handleCompareSubquery(ctx context.Context, planCtx return v, true } corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, np) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, false, np) // Only (a,b,c) = any (...) and (a,b,c) != all (...) can use row expression. canMultiCol := (!v.All && v.Op == opcode.EQ) || (v.All && v.Op == opcode.NE) @@ -1041,7 +1041,7 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, planCtx * } np = er.popExistsSubPlan(planCtx, np) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, np) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, false, np) semiJoinRewrite := hintFlags&hint.HintFlagSemiJoinRewrite > 0 if semiJoinRewrite && noDecorrelate { @@ -1213,7 +1213,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp lt, rt := lexpr.GetType(er.sctx.GetEvalCtx()), rexpr.GetType(er.sctx.GetEvalCtx()) collFlag := collate.CompatibleCollate(lt.GetCollate(), rt.GetCollate()) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, np) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, false, np) // If it's not the form of `not in (SUBQUERY)`, // and has no correlated column from the current level plan(if the correlated column is from upper level, @@ -1265,7 +1265,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp return v, true } -func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64, subqueryPlan base.LogicalPlan) bool { +func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64, allowVarOverride bool, subqueryPlan base.LogicalPlan) bool { noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 if noDecorrelate { @@ -1274,7 +1274,8 @@ func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.Correla "NO_DECORRELATE() is inapplicable because there are no correlated columns.") noDecorrelate = false } - } else if len(corCols) > 0 && planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate && subqueryPlan != nil { + } else if allowVarOverride && // Only scalar subquery allows the variable override (true) currently + len(corCols) > 0 && planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate && subqueryPlan != nil { tableInfo := getTableInfoFromPlan(subqueryPlan) if tableInfo != nil && canEfficientlyEvaluateCorrelation(corCols, tableInfo) { // Only enable decorrelation by the session variable if correlation columns are index-covered @@ -1354,7 +1355,7 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, planCtx } np = planCtx.builder.buildMaxOneRow(np) correlatedColumn := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags, np) + noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags, true, np) if planCtx.builder.disableSubQueryPreprocessing || len(coreusage.ExtractCorrelatedCols4LogicalPlan(np)) > 0 || hasCTEConsumerInSubPlan(np) { planCtx.plan = planCtx.builder.buildApplyWithJoinType(planCtx.plan, np, logicalop.LeftOuterJoin, noDecorrelate) diff --git a/pkg/planner/core/logical_plan_builder.go b/pkg/planner/core/logical_plan_builder.go index f72198d87e671..746f80aeb9f31 100644 --- a/pkg/planner/core/logical_plan_builder.go +++ b/pkg/planner/core/logical_plan_builder.go @@ -2753,7 +2753,7 @@ func (r *correlatedAggregateResolver) collectFromGroupBy(p base.LogicalPlan, gro r.b.curClause = groupByClause outerAggFuncs, err := r.b.extractCorrelatedAggFuncs(r.ctx, p, aggList) if err != nil { - return err + return nil } r.correlatedAggFuncs = append(r.correlatedAggFuncs, outerAggFuncs...) return nil @@ -5831,11 +5831,11 @@ func (b *PlanBuilder) buildUpdateLists(ctx context.Context, tableList []*ast.Tab newList = make([]*expression.Assignment, 0, p.Schema().Len()) tblDbMap := make(map[string]string, len(tableList)) for _, tbl := range tableList { - tnW := b.resolveCtx.GetTableName(tbl) - if isCTE(tnW) { + tblW := b.resolveCtx.GetTableName(tbl) + if isCTE(tblW) { continue } - tblDbMap[tbl.Name.L] = tnW.DBInfo.Name.L + tblDbMap[tbl.Name.L] = tblW.DBInfo.Name.L } allAssignments := append(list, virtualAssignments...) @@ -6381,6 +6381,17 @@ func (b *PlanBuilder) buildWindowFunctionFrameBound(_ context.Context, spec *ast return bound, nil } + bound.CalcFuncs = make([]expression.Expression, len(orderByItems)) + bound.CmpFuncs = make([]expression.CompareFunc, len(orderByItems)) + if bound.Type == ast.CurrentRow { + for i, item := range orderByItems { + col := item.Col + bound.CalcFuncs[i] = col + bound.CmpFuncs[i] = expression.GetCmpFunction(b.ctx.GetExprCtx(), col, col) + } + return bound, nil + } + col := orderByItems[0].Col // TODO: We also need to raise error for non-deterministic expressions, like rand(). val, err := evalAstExprWithPlanCtx(b.ctx, boundClause.Expr) From c4745640a19cf805e0f380fdb015d1ec59210ac8 Mon Sep 17 00:00:00 2001 From: tpp Date: Wed, 27 Aug 2025 17:38:22 -0700 Subject: [PATCH 06/18] comprehensive refactor4 --- pkg/planner/core/expression_rewriter.go | 34 ++++++++++++++----------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index 7fc1237d9e66a..5dca422858e65 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -1041,7 +1041,7 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, planCtx * } np = er.popExistsSubPlan(planCtx, np) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, false, np) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, true, np) semiJoinRewrite := hintFlags&hint.HintFlagSemiJoinRewrite > 0 if semiJoinRewrite && noDecorrelate { @@ -1274,7 +1274,7 @@ func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.Correla "NO_DECORRELATE() is inapplicable because there are no correlated columns.") noDecorrelate = false } - } else if allowVarOverride && // Only scalar subquery allows the variable override (true) currently + } else if allowVarOverride && // Only scalar and exists subquery allows the variable override (true) currently len(corCols) > 0 && planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate && subqueryPlan != nil { tableInfo := getTableInfoFromPlan(subqueryPlan) if tableInfo != nil && canEfficientlyEvaluateCorrelation(corCols, tableInfo) { @@ -1292,9 +1292,15 @@ func getTableInfoFromPlan(p base.LogicalPlan) *model.TableInfo { switch v := p.(type) { case *logicalop.DataSource: return v.TableInfo - case *logicalop.LogicalProjection, *logicalop.LogicalSelection, *logicalop.LogicalAggregation, *logicalop.LogicalSort, *logicalop.LogicalLimit, *logicalop.LogicalMaxOneRow: - if len(v.Children()) > 0 { - return getTableInfoFromPlan(v.Children()[0]) + case *logicalop.LogicalProjection, *logicalop.LogicalSelection, *logicalop.LogicalAggregation, + *logicalop.LogicalSort, *logicalop.LogicalLimit, *logicalop.LogicalMaxOneRow, + *logicalop.LogicalJoin, *logicalop.LogicalUnionAll, *logicalop.LogicalWindow, + *logicalop.LogicalTopN, *logicalop.LogicalExpand, *logicalop.LogicalSequence: + // Check ALL children for table info + for _, child := range v.Children() { + if tableInfo := getTableInfoFromPlan(child); tableInfo != nil { + return tableInfo + } } } return nil @@ -1312,22 +1318,20 @@ func canEfficientlyEvaluateCorrelation(corCols []*expression.CorrelatedColumn, t corColIDs[corCol.Column.UniqueID] = true } - // Check if any index covers these correlation columns + // Check if any index has correlation columns as the first column for _, idx := range tableInfo.Indices { if idx.State != model.StatePublic { continue } - // Check if index prefix covers correlation columns - for i, col := range idx.Columns { - // Get the actual column ID from the table info using the offset - if col.Offset < len(tableInfo.Columns) { - colID := tableInfo.Columns[col.Offset].ID + // Only check the first column of each index + if len(idx.Columns) > 0 { + firstCol := idx.Columns[0] + if firstCol.Offset < len(tableInfo.Columns) { + colID := tableInfo.Columns[firstCol.Offset].ID if corColIDs[colID] { - // Found a correlation column in index, check if it's in prefix - if i == 0 || (i > 0 && idx.Columns[i-1].Offset != col.Offset) { - return true - } + // Found a correlation column as the first column of an index + return true } } } From 7644cea26362d1c973035808c9149fe581a6eb8e Mon Sep 17 00:00:00 2001 From: tpp Date: Thu, 28 Aug 2025 11:45:39 -0700 Subject: [PATCH 07/18] comprehensive refactor5 --- pkg/planner/core/expression_rewriter.go | 52 ++++++++++++++++--------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index 5dca422858e65..2ce6d029bfaf6 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -257,7 +257,7 @@ func (b *PlanBuilder) getExpressionRewriter(ctx context.Context, p base.LogicalP if len(b.rewriterPool) < b.rewriterCounter { rewriter = &expressionRewriter{ sctx: b.ctx.GetExprCtx(), ctx: ctx, - planCtx: &exprRewriterPlanCtx{plan: p, builder: b, rollExpand: b.currentBlockExpand}, + planCtx: &exprRewriterPlanCtx{plan: p, builder: b, curClause: b.curClause, rollExpand: b.currentBlockExpand}, } b.rewriterPool = append(b.rewriterPool, rewriter) return @@ -273,6 +273,7 @@ func (b *PlanBuilder) getExpressionRewriter(ctx context.Context, p base.LogicalP rewriter.ctx = ctx rewriter.err = nil rewriter.planCtx.plan = p + rewriter.planCtx.curClause = b.curClause rewriter.planCtx.aggrMap = nil rewriter.planCtx.insertPlan = nil rewriter.planCtx.rollExpand = b.currentBlockExpand @@ -330,6 +331,9 @@ type exprRewriterPlanCtx struct { plan base.LogicalPlan builder *PlanBuilder + // curClause tracks which part of the query is being processed + curClause clauseCode + aggrMap map[*ast.AggregateFuncExpr]int windowMap map[*ast.WindowFuncExpr]int @@ -1274,11 +1278,23 @@ func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.Correla "NO_DECORRELATE() is inapplicable because there are no correlated columns.") noDecorrelate = false } - } else if allowVarOverride && // Only scalar and exists subquery allows the variable override (true) currently - len(corCols) > 0 && planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate && subqueryPlan != nil { - tableInfo := getTableInfoFromPlan(subqueryPlan) - if tableInfo != nil && canEfficientlyEvaluateCorrelation(corCols, tableInfo) { - // Only enable decorrelation by the session variable if correlation columns are index-covered + } else if allowVarOverride && + len(corCols) > 0 && + planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate && + subqueryPlan != nil { + + // Check if this is a select list subquery + isSelectListSubquery := planCtx.curClause == fieldList + + if isSelectListSubquery { + // For select list subqueries, apply index coverage check + tableInfo := getTableInfoFromPlan(subqueryPlan) + if tableInfo != nil && canEfficientlyEvaluateCorrelation(corCols, tableInfo) { + // Only enable decorrelation by the session variable if correlation columns are index-covered + noDecorrelate = true + } + } else { + // For where clause subqueries, always allow decorrelation if session variable is enabled noDecorrelate = true } } @@ -1292,10 +1308,11 @@ func getTableInfoFromPlan(p base.LogicalPlan) *model.TableInfo { switch v := p.(type) { case *logicalop.DataSource: return v.TableInfo - case *logicalop.LogicalProjection, *logicalop.LogicalSelection, *logicalop.LogicalAggregation, - *logicalop.LogicalSort, *logicalop.LogicalLimit, *logicalop.LogicalMaxOneRow, - *logicalop.LogicalJoin, *logicalop.LogicalUnionAll, *logicalop.LogicalWindow, - *logicalop.LogicalTopN, *logicalop.LogicalExpand, *logicalop.LogicalSequence: + //case *logicalop.LogicalProjection, *logicalop.LogicalSelection, *logicalop.LogicalAggregation, + // *logicalop.LogicalSort, *logicalop.LogicalLimit, *logicalop.LogicalMaxOneRow, + // *logicalop.LogicalJoin, *logicalop.LogicalUnionAll, *logicalop.LogicalWindow, + // *logicalop.LogicalTopN, *logicalop.LogicalExpand, *logicalop.LogicalSequence: + default: // Check ALL children for table info for _, child := range v.Children() { if tableInfo := getTableInfoFromPlan(child); tableInfo != nil { @@ -1306,7 +1323,7 @@ func getTableInfoFromPlan(p base.LogicalPlan) *model.TableInfo { return nil } -// canEfficientlyEvaluateCorrelation checks if correlation columns are covered by any index +// canEfficientlyEvaluateCorrelation checks if correlation columns are covered by any index (in any position) func canEfficientlyEvaluateCorrelation(corCols []*expression.CorrelatedColumn, tableInfo *model.TableInfo) bool { if len(corCols) == 0 || tableInfo == nil { return false @@ -1318,19 +1335,18 @@ func canEfficientlyEvaluateCorrelation(corCols []*expression.CorrelatedColumn, t corColIDs[corCol.Column.UniqueID] = true } - // Check if any index has correlation columns as the first column + // Check if any index has correlation columns in any position for _, idx := range tableInfo.Indices { if idx.State != model.StatePublic { continue } - // Only check the first column of each index - if len(idx.Columns) > 0 { - firstCol := idx.Columns[0] - if firstCol.Offset < len(tableInfo.Columns) { - colID := tableInfo.Columns[firstCol.Offset].ID + // Check all columns in the index, not just the first one + for _, col := range idx.Columns { + if col.Offset < len(tableInfo.Columns) { + colID := tableInfo.Columns[col.Offset].ID if corColIDs[colID] { - // Found a correlation column as the first column of an index + // Found a correlation column in the index return true } } From de3ec63edd937ac7d7b9eeadaa9cdeb82c230429 Mon Sep 17 00:00:00 2001 From: tpp Date: Thu, 28 Aug 2025 13:53:18 -0700 Subject: [PATCH 08/18] remove index check --- pkg/planner/core/expression_rewriter.go | 79 +------------------------ 1 file changed, 2 insertions(+), 77 deletions(-) diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index 2ce6d029bfaf6..e6b00d7fc4c8f 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -1280,90 +1280,15 @@ func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.Correla } } else if allowVarOverride && len(corCols) > 0 && - planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate && - subqueryPlan != nil { - + planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate { // Check if this is a select list subquery - isSelectListSubquery := planCtx.curClause == fieldList - - if isSelectListSubquery { - // For select list subqueries, apply index coverage check - tableInfo := getTableInfoFromPlan(subqueryPlan) - if tableInfo != nil && canEfficientlyEvaluateCorrelation(corCols, tableInfo) { - // Only enable decorrelation by the session variable if correlation columns are index-covered - noDecorrelate = true - } - } else { - // For where clause subqueries, always allow decorrelation if session variable is enabled + if planCtx.curClause == fieldList { noDecorrelate = true } } - return noDecorrelate } -// getTableInfoFromPlan extracts table info from a logical plan -func getTableInfoFromPlan(p base.LogicalPlan) *model.TableInfo { - // Try to find table info by traversing the plan tree - switch v := p.(type) { - case *logicalop.DataSource: - return v.TableInfo - //case *logicalop.LogicalProjection, *logicalop.LogicalSelection, *logicalop.LogicalAggregation, - // *logicalop.LogicalSort, *logicalop.LogicalLimit, *logicalop.LogicalMaxOneRow, - // *logicalop.LogicalJoin, *logicalop.LogicalUnionAll, *logicalop.LogicalWindow, - // *logicalop.LogicalTopN, *logicalop.LogicalExpand, *logicalop.LogicalSequence: - default: - // Check ALL children for table info - for _, child := range v.Children() { - if tableInfo := getTableInfoFromPlan(child); tableInfo != nil { - return tableInfo - } - } - } - return nil -} - -// canEfficientlyEvaluateCorrelation checks if correlation columns are covered by any index (in any position) -func canEfficientlyEvaluateCorrelation(corCols []*expression.CorrelatedColumn, tableInfo *model.TableInfo) bool { - if len(corCols) == 0 || tableInfo == nil { - return false - } - - // Extract column IDs from correlation columns - corColIDs := make(map[int64]bool) - for _, corCol := range corCols { - corColIDs[corCol.Column.UniqueID] = true - } - - // Check if any index has correlation columns in any position - for _, idx := range tableInfo.Indices { - if idx.State != model.StatePublic { - continue - } - - // Check all columns in the index, not just the first one - for _, col := range idx.Columns { - if col.Offset < len(tableInfo.Columns) { - colID := tableInfo.Columns[col.Offset].ID - if corColIDs[colID] { - // Found a correlation column in the index - return true - } - } - } - } - - // Check primary key coverage - if tableInfo.PKIsHandle { - pkCol := tableInfo.GetPkColInfo() - if pkCol != nil && corColIDs[pkCol.ID] { - return true - } - } - - return false -} - func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, planCtx *exprRewriterPlanCtx, v *ast.SubqueryExpr) (ast.Node, bool) { intest.AssertNotNil(planCtx) ci := planCtx.builder.prepareCTECheckForSubQuery() From 5f0e63b9b6451139ba26e79c80d663ae9699d311 Mon Sep 17 00:00:00 2001 From: tpp Date: Thu, 28 Aug 2025 14:32:52 -0700 Subject: [PATCH 09/18] build error --- pkg/planner/core/expression_rewriter.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index e6b00d7fc4c8f..213811f860d0b 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -739,7 +739,7 @@ func (er *expressionRewriter) handleCompareSubquery(ctx context.Context, planCtx return v, true } corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, false, np) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, false) // Only (a,b,c) = any (...) and (a,b,c) != all (...) can use row expression. canMultiCol := (!v.All && v.Op == opcode.EQ) || (v.All && v.Op == opcode.NE) @@ -1045,7 +1045,7 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, planCtx * } np = er.popExistsSubPlan(planCtx, np) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, true, np) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, true) semiJoinRewrite := hintFlags&hint.HintFlagSemiJoinRewrite > 0 if semiJoinRewrite && noDecorrelate { @@ -1217,7 +1217,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp lt, rt := lexpr.GetType(er.sctx.GetEvalCtx()), rexpr.GetType(er.sctx.GetEvalCtx()) collFlag := collate.CompatibleCollate(lt.GetCollate(), rt.GetCollate()) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, false, np) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, false) // If it's not the form of `not in (SUBQUERY)`, // and has no correlated column from the current level plan(if the correlated column is from upper level, @@ -1269,7 +1269,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp return v, true } -func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64, allowVarOverride bool, subqueryPlan base.LogicalPlan) bool { +func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64, allowVarOverride bool) bool { noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 if noDecorrelate { @@ -1300,7 +1300,7 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, planCtx } np = planCtx.builder.buildMaxOneRow(np) correlatedColumn := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags, true, np) + noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags, true) if planCtx.builder.disableSubQueryPreprocessing || len(coreusage.ExtractCorrelatedCols4LogicalPlan(np)) > 0 || hasCTEConsumerInSubPlan(np) { planCtx.plan = planCtx.builder.buildApplyWithJoinType(planCtx.plan, np, logicalop.LeftOuterJoin, noDecorrelate) From 93eb561eafb981c09ee0d1b34fadc1d7e3f8ef22 Mon Sep 17 00:00:00 2001 From: tpp Date: Thu, 28 Aug 2025 17:02:26 -0700 Subject: [PATCH 10/18] rename variable --- pkg/bindinfo/binding_plan_generation.go | 10 ++++++++-- pkg/bindinfo/binding_plan_generation_test.go | 4 +++- pkg/planner/core/expression_rewriter.go | 12 +++++++----- pkg/planner/core/planbuilder.go | 2 +- pkg/sessionctx/vardef/tidb_vars.go | 8 ++++---- pkg/sessionctx/variable/session.go | 5 +++-- pkg/sessionctx/variable/sysvar.go | 4 ++-- 7 files changed, 28 insertions(+), 17 deletions(-) diff --git a/pkg/bindinfo/binding_plan_generation.go b/pkg/bindinfo/binding_plan_generation.go index 97f29aeee5c86..63c2bd8853584 100644 --- a/pkg/bindinfo/binding_plan_generation.go +++ b/pkg/bindinfo/binding_plan_generation.go @@ -342,6 +342,8 @@ func genPlanUnderState(sctx sessionctx.Context, stmt ast.StmtNode, state *state) sctx.GetSessionVars().RiskGroupNDVSkewRatio = state.varValues[i].(float64) case vardef.TiDBOptPreferRangeScan: sctx.GetSessionVars().SetAllowPreferRangeScan(state.varValues[i].(bool)) + case vardef.TiDBOptEnableNoDecorrelateInSelect: + sctx.GetSessionVars().EnableNoDecorrelateInSelect = state.varValues[i].(bool) default: return nil, fmt.Errorf("unsupported variable %s in plan generation", varName) } @@ -409,7 +411,7 @@ func adjustVar(varName string, varVal any) (newVarVal any, err error) { return v, nil } return v * 5, nil - case vardef.TiDBOptOrderingIdxSelRatio, vardef.TiDBOptRiskEqSkewRatio, vardef.TiDBOptRiskGroupNDVSkewRatio: // range [0, 1], "<=0" means disable + case vardef.TiDBOptOrderingIdxSelRatio, vardef.TiDBOptRiskEqSkewRatio, vardef.TiDBOptRiskRangeSkewRatio, vardef.TiDBOptRiskGroupNDVSkewRatio: // range [0, 1], "<=0" means disable v := varVal.(float64) if v <= 0 { return 0.1, nil @@ -418,7 +420,7 @@ func adjustVar(varName string, varVal any) (newVarVal any, err error) { } // increase 0.1 each step return v + 0.1, nil - case vardef.TiDBOptPreferRangeScan: // flip the switch + case vardef.TiDBOptPreferRangeScan, vardef.TiDBOptEnableNoDecorrelateInSelect: // flip the switch return !varVal.(bool), nil } return nil, fmt.Errorf("unsupported variable %s in plan generation", varName) @@ -491,10 +493,14 @@ func getStartState(vars []string, fixes []uint64) (*state, error) { s.varValues = append(s.varValues, vardef.DefTiDBOptOrderingIdxSelRatio) case vardef.TiDBOptRiskEqSkewRatio: s.varValues = append(s.varValues, vardef.DefOptRiskEqSkewRatio) + case vardef.TiDBOptRiskRangeSkewRatio: + s.varValues = append(s.varValues, vardef.DefOptRiskRangeSkewRatio) case vardef.TiDBOptRiskGroupNDVSkewRatio: s.varValues = append(s.varValues, vardef.DefOptRiskGroupNDVSkewRatio) case vardef.TiDBOptPreferRangeScan: s.varValues = append(s.varValues, vardef.DefOptPreferRangeScan) + case vardef.TiDBOptEnableNoDecorrelateInSelect: + s.varValues = append(s.varValues, vardef.DefOptEnableNoDecorrelateInSelect) default: return nil, fmt.Errorf("unsupported variable %s in plan generation", varName) } diff --git a/pkg/bindinfo/binding_plan_generation_test.go b/pkg/bindinfo/binding_plan_generation_test.go index 3b0571eea32c9..742f54d97a331 100644 --- a/pkg/bindinfo/binding_plan_generation_test.go +++ b/pkg/bindinfo/binding_plan_generation_test.go @@ -99,10 +99,12 @@ func TestStartState(t *testing.T) { vardef.TiDBOptIndexJoinCostFactor, vardef.TiDBOptOrderingIdxSelRatio, vardef.TiDBOptRiskEqSkewRatio, + vardef.TiDBOptPreferRangeScan, + vardef.TiDBOptEnableNoDecorrelateInSelect, } fixes := []uint64{fixcontrol.Fix44855, fixcontrol.Fix45132, fixcontrol.Fix52869} state, err := getStartState(vars, fixes) require.NoError(t, err) - require.Equal(t, state.Encode(), "1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,0.0100,0.0000,OFF,1000,OFF") + require.Equal(t, state.Encode(), "1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,0.0100,0.0000,ON,OFF,OFF,1000,OFF") } diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index 213811f860d0b..8224b3e8ccccf 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -1278,11 +1278,13 @@ func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.Correla "NO_DECORRELATE() is inapplicable because there are no correlated columns.") noDecorrelate = false } - } else if allowVarOverride && - len(corCols) > 0 && - planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelate { - // Check if this is a select list subquery - if planCtx.curClause == fieldList { + } + if allowVarOverride && // caller passed true to allow EnableNoDecorrelateInSelect variable to apply + len(corCols) > 0 && // has correlation columns + planCtx.curClause == fieldList { // subquery is in the select list + planCtx.builder.ctx.GetSessionVars().RecordRelevantOptVar(vardef.TiDBOptEnableNoDecorrelateInSelect) + // If it isn't already enabled via hint, and variable is set, then enable it + if !noDecorrelate && planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelateInSelect { noDecorrelate = true } } diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index 930c7b8f026c7..981140efec164 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -465,7 +465,7 @@ func (b *PlanBuilder) Init(sctx base.PlanContext, is infoschema.InfoSchema, proc b.is = is b.hintProcessor = processor b.isForUpdateRead = sctx.GetSessionVars().IsPessimisticReadConsistency() - b.noDecorrelate = sctx.GetSessionVars().EnableNoDecorrelate + b.noDecorrelate = sctx.GetSessionVars().EnableNoDecorrelateInSelect if savedBlockNames == nil { return b, nil } diff --git a/pkg/sessionctx/vardef/tidb_vars.go b/pkg/sessionctx/vardef/tidb_vars.go index 6122e2ff40d05..03e7742880bc3 100644 --- a/pkg/sessionctx/vardef/tidb_vars.go +++ b/pkg/sessionctx/vardef/tidb_vars.go @@ -312,6 +312,9 @@ const ( // TiDBOptPreferRangeScan is used to enable/disable the optimizer to always prefer range scan over table scan, ignoring their costs. TiDBOptPreferRangeScan = "tidb_opt_prefer_range_scan" + // TiDBOptEnableNoDecorrelateInSelect is used to control whether to enable the NO_DECORRELATE hint for subqueries in the select list. + TiDBOptEnableNoDecorrelateInSelect = "tidb_opt_enable_no_decorrelate_in_select" + // TiDBOptEnableCorrelationAdjustment is used to indicates if enable correlation adjustment. TiDBOptEnableCorrelationAdjustment = "tidb_opt_enable_correlation_adjustment" @@ -598,9 +601,6 @@ const ( // TiDBEnablePipelinedWindowFunction is used to control whether to use pipelined window function, it only works when tidb_enable_window_function = true. TiDBEnablePipelinedWindowFunction = "tidb_enable_pipelined_window_function" - // TiDBEnableNoDecorrelate is used to control whether to enable the NO_DECORRELATE hint globally or at session level. - TiDBEnableNoDecorrelate = "tidb_enable_no_decorrelate" - // TiDBEnableStrictDoubleTypeCheck is used to control table field double type syntax check. TiDBEnableStrictDoubleTypeCheck = "tidb_enable_strict_double_type_check" @@ -1377,6 +1377,7 @@ const ( DefOptForceInlineCTE = false DefOptInSubqToJoinAndAgg = true DefOptPreferRangeScan = true + DefOptEnableNoDecorrelateInSelect = false DefBatchInsert = false DefBatchDelete = false DefBatchCommit = false @@ -1440,7 +1441,6 @@ const ( DefTiDBForcePriority = mysql.NoPriority DefEnableWindowFunction = true DefEnablePipelinedWindowFunction = true - DefEnableNoDecorrelate = false DefEnableStrictDoubleTypeCheck = true DefEnableVectorizedExpression = true DefTiDBOptJoinReorderThreshold = 0 diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index 62e40874d8adb..a2a4d6690e3dd 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -1133,8 +1133,8 @@ type SessionVars struct { // EnablePipelinedWindowExec enables executing window functions in a pipelined manner. EnablePipelinedWindowExec bool - // EnableNoDecorrelate enables the NO_DECORRELATE hint globally or at session level. - EnableNoDecorrelate bool + // EnableNoDecorrelateInSelect enables the NO_DECORRELATE hint for subqueries in the select list. + EnableNoDecorrelateInSelect bool // AllowProjectionPushDown enables pushdown projection on TiKV. AllowProjectionPushDown bool @@ -2201,6 +2201,7 @@ func NewSessionVars(hctx HookContext) *SessionVars { OptimizerSelectivityLevel: vardef.DefTiDBOptimizerSelectivityLevel, RiskGroupNDVSkewRatio: vardef.DefOptRiskGroupNDVSkewRatio, EnableOuterJoinReorder: vardef.DefTiDBEnableOuterJoinReorder, + EnableNoDecorrelateInSelect: vardef.DefOptEnableNoDecorrelateInSelect, RetryLimit: vardef.DefTiDBRetryLimit, DisableTxnAutoRetry: vardef.DefTiDBDisableTxnAutoRetry, DDLReorgPriority: kv.PriorityLow, diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index e0f4344d9d27c..350466559762a 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -2356,8 +2356,8 @@ var defaultSysVars = []*SysVar{ s.EnablePipelinedWindowExec = TiDBOptOn(val) return nil }}, - {Scope: vardef.ScopeGlobal | vardef.ScopeSession, Name: vardef.TiDBEnableNoDecorrelate, Value: BoolToOnOff(vardef.DefEnableNoDecorrelate), Type: vardef.TypeBool, SetSession: func(s *SessionVars, val string) error { - s.EnableNoDecorrelate = TiDBOptOn(val) + {Scope: vardef.ScopeGlobal | vardef.ScopeSession, Name: vardef.TiDBOptEnableNoDecorrelateInSelect, Value: BoolToOnOff(vardef.DefOptEnableNoDecorrelateInSelect), Type: vardef.TypeBool, SetSession: func(s *SessionVars, val string) error { + s.EnableNoDecorrelateInSelect = TiDBOptOn(val) return nil }}, {Scope: vardef.ScopeGlobal | vardef.ScopeSession, Name: vardef.TiDBEnableStrictDoubleTypeCheck, Value: BoolToOnOff(vardef.DefEnableStrictDoubleTypeCheck), Type: vardef.TypeBool, SetSession: func(s *SessionVars, val string) error { From ede91144d97a092697e093de1d66c37ea622e569 Mon Sep 17 00:00:00 2001 From: tpp Date: Thu, 28 Aug 2025 18:11:45 -0700 Subject: [PATCH 11/18] minor reformat --- pkg/planner/core/expression_rewriter.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index 8224b3e8ccccf..ecdaf9bf6149d 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -1272,15 +1272,13 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64, allowVarOverride bool) bool { noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 - if noDecorrelate { - if len(corCols) == 0 { + if len(corCols) == 0 { + if noDecorrelate { planCtx.builder.ctx.GetSessionVars().StmtCtx.SetHintWarning( "NO_DECORRELATE() is inapplicable because there are no correlated columns.") noDecorrelate = false } - } - if allowVarOverride && // caller passed true to allow EnableNoDecorrelateInSelect variable to apply - len(corCols) > 0 && // has correlation columns + } else if allowVarOverride && // caller passed true to allow EnableNoDecorrelateInSelect variable to apply planCtx.curClause == fieldList { // subquery is in the select list planCtx.builder.ctx.GetSessionVars().RecordRelevantOptVar(vardef.TiDBOptEnableNoDecorrelateInSelect) // If it isn't already enabled via hint, and variable is set, then enable it From be6c617ff43fc95f6c8da41e86a2573d2b1c9ffd Mon Sep 17 00:00:00 2001 From: tpp Date: Thu, 28 Aug 2025 18:18:45 -0700 Subject: [PATCH 12/18] minor reformat2 --- pkg/bindinfo/binding_plan_generation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bindinfo/binding_plan_generation_test.go b/pkg/bindinfo/binding_plan_generation_test.go index 45c88f0fc1c3a..6d1ea03db211f 100644 --- a/pkg/bindinfo/binding_plan_generation_test.go +++ b/pkg/bindinfo/binding_plan_generation_test.go @@ -102,7 +102,7 @@ func TestStartState(t *testing.T) { vardef.TiDBOptRiskRangeSkewRatio, vardef.TiDBOptRiskGroupNDVSkewRatio, vardef.TiDBOptSelectivityFactor, - vardef.TiDBOptPreferRangeScan, + vardef.TiDBOptPreferRangeScan, vardef.TiDBOptEnableNoDecorrelateInSelect, } fixes := []uint64{fixcontrol.Fix44855, fixcontrol.Fix45132, fixcontrol.Fix52869} From 9a0f9eed1bc4b77c0638d73c0aeee2f2f1bb5d74 Mon Sep 17 00:00:00 2001 From: tpp Date: Thu, 28 Aug 2025 22:19:44 -0700 Subject: [PATCH 13/18] testcase1 --- pkg/bindinfo/binding_plan_generation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bindinfo/binding_plan_generation_test.go b/pkg/bindinfo/binding_plan_generation_test.go index 6d1ea03db211f..7a4d141aa34a7 100644 --- a/pkg/bindinfo/binding_plan_generation_test.go +++ b/pkg/bindinfo/binding_plan_generation_test.go @@ -109,5 +109,5 @@ func TestStartState(t *testing.T) { state, err := getStartState(vars, fixes) require.NoError(t, err) - require.Equal(t, state.Encode(), "1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,0.0100,0.0000,0.0000,0.0000,0.8000,ON,OFF,OFF,1000,OFF") + require.Equal(t, state.Encode(), "1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,0.0100,0.0000,0.0000,0.0000,0.8000,true,false,OFF,1000,OFF") } From dc11377823ddf02b417b4462f26b511e3f9034f7 Mon Sep 17 00:00:00 2001 From: tpp Date: Fri, 29 Aug 2025 12:12:34 -0700 Subject: [PATCH 14/18] simplify parameter passing --- pkg/planner/core/expression_rewriter.go | 26 ++++++++++++++---------- pkg/sessionctx/variable/setvar_affect.go | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index ecdaf9bf6149d..8306387615b30 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -739,7 +739,7 @@ func (er *expressionRewriter) handleCompareSubquery(ctx context.Context, planCtx return v, true } corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, false) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags) // Only (a,b,c) = any (...) and (a,b,c) != all (...) can use row expression. canMultiCol := (!v.All && v.Op == opcode.EQ) || (v.All && v.Op == opcode.NE) @@ -1045,7 +1045,7 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, planCtx * } np = er.popExistsSubPlan(planCtx, np) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, true) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags) semiJoinRewrite := hintFlags&hint.HintFlagSemiJoinRewrite > 0 if semiJoinRewrite && noDecorrelate { @@ -1217,7 +1217,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp lt, rt := lexpr.GetType(er.sctx.GetEvalCtx()), rexpr.GetType(er.sctx.GetEvalCtx()) collFlag := collate.CompatibleCollate(lt.GetCollate(), rt.GetCollate()) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, false) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags) // If it's not the form of `not in (SUBQUERY)`, // and has no correlated column from the current level plan(if the correlated column is from upper level, @@ -1269,7 +1269,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp return v, true } -func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64, allowVarOverride bool) bool { +func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64) bool { noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 if len(corCols) == 0 { @@ -1278,12 +1278,16 @@ func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.Correla "NO_DECORRELATE() is inapplicable because there are no correlated columns.") noDecorrelate = false } - } else if allowVarOverride && // caller passed true to allow EnableNoDecorrelateInSelect variable to apply - planCtx.curClause == fieldList { // subquery is in the select list - planCtx.builder.ctx.GetSessionVars().RecordRelevantOptVar(vardef.TiDBOptEnableNoDecorrelateInSelect) - // If it isn't already enabled via hint, and variable is set, then enable it - if !noDecorrelate && planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelateInSelect { - noDecorrelate = true + } else { + sctx := planCtx.builder.subQueryCtx + // Only support scalar and exists subqueries + validSubqType := sctx == handlingScalarSubquery || sctx == handlingExistsSubquery + if validSubqType && planCtx.curClause == fieldList { // subquery is in the select list + planCtx.builder.ctx.GetSessionVars().RecordRelevantOptVar(vardef.TiDBOptEnableNoDecorrelateInSelect) + // If it isn't already enabled via hint, and variable is set, then enable it + if !noDecorrelate && planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelateInSelect { + noDecorrelate = true + } } } return noDecorrelate @@ -1300,7 +1304,7 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, planCtx } np = planCtx.builder.buildMaxOneRow(np) correlatedColumn := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags, true) + noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags) if planCtx.builder.disableSubQueryPreprocessing || len(coreusage.ExtractCorrelatedCols4LogicalPlan(np)) > 0 || hasCTEConsumerInSubPlan(np) { planCtx.plan = planCtx.builder.buildApplyWithJoinType(planCtx.plan, np, logicalop.LeftOuterJoin, noDecorrelate) diff --git a/pkg/sessionctx/variable/setvar_affect.go b/pkg/sessionctx/variable/setvar_affect.go index f15abd14f67ad..a2118288159db 100644 --- a/pkg/sessionctx/variable/setvar_affect.go +++ b/pkg/sessionctx/variable/setvar_affect.go @@ -116,6 +116,7 @@ var isHintUpdatableVerified = map[string]struct{}{ "tidb_store_batch_size": {}, "mpp_version": {}, "tidb_enable_inl_join_inner_multi_pattern": {}, + "tidb_opt_enable_no_decorrelate_in_select": {}, "tidb_opt_enable_late_materialization": {}, "tidb_opt_ordering_index_selectivity_threshold": {}, "tidb_opt_ordering_index_selectivity_ratio": {}, @@ -131,7 +132,6 @@ var isHintUpdatableVerified = map[string]struct{}{ "tiflash_fine_grained_shuffle_batch_size": {}, "tiflash_fine_grained_shuffle_stream_count": {}, "tidb_hash_join_version": {}, - "tidb_enable_no_decorrelate": {}, // Variables that is compatible with MySQL. "cte_max_recursion_depth": {}, "sql_mode": {}, From d8196261dfd4b6ae145e3fde3ca84f356d009ef5 Mon Sep 17 00:00:00 2001 From: tpp Date: Fri, 29 Aug 2025 12:41:26 -0700 Subject: [PATCH 15/18] correct parameter passing --- pkg/bindinfo/binding_plan_generation_test.go | 2 +- pkg/planner/core/expression_rewriter.go | 13 ++++++------- pkg/sessionctx/vardef/tidb_vars.go | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/bindinfo/binding_plan_generation_test.go b/pkg/bindinfo/binding_plan_generation_test.go index 7a4d141aa34a7..403e0b75dac39 100644 --- a/pkg/bindinfo/binding_plan_generation_test.go +++ b/pkg/bindinfo/binding_plan_generation_test.go @@ -109,5 +109,5 @@ func TestStartState(t *testing.T) { state, err := getStartState(vars, fixes) require.NoError(t, err) - require.Equal(t, state.Encode(), "1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,0.0100,0.0000,0.0000,0.0000,0.8000,true,false,OFF,1000,OFF") + require.Equal(t, state.Encode(), "1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,0.0100,0.0000,0.0000,0.0000,0.8000,true,true,OFF,1000,OFF") } diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index 8306387615b30..4bec672e92bc4 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -739,7 +739,7 @@ func (er *expressionRewriter) handleCompareSubquery(ctx context.Context, planCtx return v, true } corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, handlingCompareSubquery) // Only (a,b,c) = any (...) and (a,b,c) != all (...) can use row expression. canMultiCol := (!v.All && v.Op == opcode.EQ) || (v.All && v.Op == opcode.NE) @@ -1045,7 +1045,7 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, planCtx * } np = er.popExistsSubPlan(planCtx, np) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, handlingExistsSubquery) semiJoinRewrite := hintFlags&hint.HintFlagSemiJoinRewrite > 0 if semiJoinRewrite && noDecorrelate { @@ -1217,7 +1217,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp lt, rt := lexpr.GetType(er.sctx.GetEvalCtx()), rexpr.GetType(er.sctx.GetEvalCtx()) collFlag := collate.CompatibleCollate(lt.GetCollate(), rt.GetCollate()) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, handlingInSubquery) // If it's not the form of `not in (SUBQUERY)`, // and has no correlated column from the current level plan(if the correlated column is from upper level, @@ -1269,7 +1269,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp return v, true } -func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64) bool { +func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64, sCtx subQueryCtx) bool { noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 if len(corCols) == 0 { @@ -1279,9 +1279,8 @@ func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.Correla noDecorrelate = false } } else { - sctx := planCtx.builder.subQueryCtx // Only support scalar and exists subqueries - validSubqType := sctx == handlingScalarSubquery || sctx == handlingExistsSubquery + validSubqType := sCtx == handlingScalarSubquery || sCtx == handlingExistsSubquery if validSubqType && planCtx.curClause == fieldList { // subquery is in the select list planCtx.builder.ctx.GetSessionVars().RecordRelevantOptVar(vardef.TiDBOptEnableNoDecorrelateInSelect) // If it isn't already enabled via hint, and variable is set, then enable it @@ -1304,7 +1303,7 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, planCtx } np = planCtx.builder.buildMaxOneRow(np) correlatedColumn := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags) + noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags, handlingScalarSubquery) if planCtx.builder.disableSubQueryPreprocessing || len(coreusage.ExtractCorrelatedCols4LogicalPlan(np)) > 0 || hasCTEConsumerInSubPlan(np) { planCtx.plan = planCtx.builder.buildApplyWithJoinType(planCtx.plan, np, logicalop.LeftOuterJoin, noDecorrelate) diff --git a/pkg/sessionctx/vardef/tidb_vars.go b/pkg/sessionctx/vardef/tidb_vars.go index d4369f783a3e5..c0b08866f3255 100644 --- a/pkg/sessionctx/vardef/tidb_vars.go +++ b/pkg/sessionctx/vardef/tidb_vars.go @@ -1388,7 +1388,7 @@ const ( DefOptForceInlineCTE = false DefOptInSubqToJoinAndAgg = true DefOptPreferRangeScan = true - DefOptEnableNoDecorrelateInSelect = false + DefOptEnableNoDecorrelateInSelect = true DefBatchInsert = false DefBatchDelete = false DefBatchCommit = false From e2b528cf8d475eb53d958ff9c75f84ec8debd700 Mon Sep 17 00:00:00 2001 From: tpp Date: Fri, 29 Aug 2025 15:57:46 -0700 Subject: [PATCH 16/18] revert variable default --- pkg/sessionctx/vardef/tidb_vars.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sessionctx/vardef/tidb_vars.go b/pkg/sessionctx/vardef/tidb_vars.go index c0b08866f3255..d4369f783a3e5 100644 --- a/pkg/sessionctx/vardef/tidb_vars.go +++ b/pkg/sessionctx/vardef/tidb_vars.go @@ -1388,7 +1388,7 @@ const ( DefOptForceInlineCTE = false DefOptInSubqToJoinAndAgg = true DefOptPreferRangeScan = true - DefOptEnableNoDecorrelateInSelect = true + DefOptEnableNoDecorrelateInSelect = false DefBatchInsert = false DefBatchDelete = false DefBatchCommit = false From bbb31290b245186f7e44d36e877fd36900d49adb Mon Sep 17 00:00:00 2001 From: tpp <146148086+terry1purcell@users.noreply.github.com> Date: Fri, 29 Aug 2025 17:35:48 -0700 Subject: [PATCH 17/18] Update expected state encoding in tests --- pkg/bindinfo/binding_plan_generation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/bindinfo/binding_plan_generation_test.go b/pkg/bindinfo/binding_plan_generation_test.go index 403e0b75dac39..7a4d141aa34a7 100644 --- a/pkg/bindinfo/binding_plan_generation_test.go +++ b/pkg/bindinfo/binding_plan_generation_test.go @@ -109,5 +109,5 @@ func TestStartState(t *testing.T) { state, err := getStartState(vars, fixes) require.NoError(t, err) - require.Equal(t, state.Encode(), "1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,0.0100,0.0000,0.0000,0.0000,0.8000,true,true,OFF,1000,OFF") + require.Equal(t, state.Encode(), "1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,0.0100,0.0000,0.0000,0.0000,0.8000,true,false,OFF,1000,OFF") } From 4c1daba8592f842dd858e9be92c2e73dbf34dbae Mon Sep 17 00:00:00 2001 From: tpp Date: Sat, 30 Aug 2025 17:38:23 -0700 Subject: [PATCH 18/18] resolve conflict with semijoinrewrite hint --- pkg/planner/core/expression_rewriter.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index 4bec672e92bc4..062eb1cb1bb5c 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -1279,13 +1279,18 @@ func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.Correla noDecorrelate = false } } else { - // Only support scalar and exists subqueries - validSubqType := sCtx == handlingScalarSubquery || sCtx == handlingExistsSubquery - if validSubqType && planCtx.curClause == fieldList { // subquery is in the select list - planCtx.builder.ctx.GetSessionVars().RecordRelevantOptVar(vardef.TiDBOptEnableNoDecorrelateInSelect) - // If it isn't already enabled via hint, and variable is set, then enable it - if !noDecorrelate && planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelateInSelect { - noDecorrelate = true + semiJoinRewrite := hintFlags&hint.HintFlagSemiJoinRewrite > 0 + // We can't override noDecorrelate via the variable for EXISTS subqueries with semi join rewrite + // as this will cause a conflict that will result in both being disabled in later code + if !(semiJoinRewrite && sCtx == handlingExistsSubquery) { + // Only support scalar and exists subqueries + validSubqType := sCtx == handlingScalarSubquery || sCtx == handlingExistsSubquery + if validSubqType && planCtx.curClause == fieldList { // subquery is in the select list + planCtx.builder.ctx.GetSessionVars().RecordRelevantOptVar(vardef.TiDBOptEnableNoDecorrelateInSelect) + // If it isn't already enabled via hint, and variable is set, then enable it + if !noDecorrelate && planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelateInSelect { + noDecorrelate = true + } } } }