diff --git a/pkg/bindinfo/binding_plan_generation.go b/pkg/bindinfo/binding_plan_generation.go index de0099336959c..af53d351db914 100644 --- a/pkg/bindinfo/binding_plan_generation.go +++ b/pkg/bindinfo/binding_plan_generation.go @@ -344,6 +344,8 @@ func genPlanUnderState(sctx sessionctx.Context, stmt ast.StmtNode, state *state) sctx.GetSessionVars().RiskRangeSkewRatio = state.varValues[i].(float64) case vardef.TiDBOptPreferRangeScan: sctx.GetSessionVars().SetAllowPreferRangeScan(state.varValues[i].(bool)) + case vardef.TiDBOptEnableNoDecorrelateInSelect: + sctx.GetSessionVars().EnableNoDecorrelateInSelect = state.varValues[i].(bool) case vardef.TiDBOptSelectivityFactor: sctx.GetSessionVars().SelectivityFactor = state.varValues[i].(float64) default: @@ -422,7 +424,7 @@ func adjustVar(varName string, varVal any) (newVarVal any, err error) { } // increase 0.1 each step return v + 0.1, nil - case vardef.TiDBOptPreferRangeScan: // flip the switch + case vardef.TiDBOptPreferRangeScan, vardef.TiDBOptEnableNoDecorrelateInSelect: // flip the switch return !varVal.(bool), nil } return nil, fmt.Errorf("unsupported variable %s in plan generation", varName) @@ -501,6 +503,8 @@ func getStartState(vars []string, fixes []uint64) (*state, error) { s.varValues = append(s.varValues, vardef.DefOptRiskGroupNDVSkewRatio) case vardef.TiDBOptPreferRangeScan: s.varValues = append(s.varValues, vardef.DefOptPreferRangeScan) + case vardef.TiDBOptEnableNoDecorrelateInSelect: + s.varValues = append(s.varValues, vardef.DefOptEnableNoDecorrelateInSelect) case vardef.TiDBOptSelectivityFactor: s.varValues = append(s.varValues, vardef.DefOptSelectivityFactor) default: diff --git a/pkg/bindinfo/binding_plan_generation_test.go b/pkg/bindinfo/binding_plan_generation_test.go index 2696a68618fa5..7a4d141aa34a7 100644 --- a/pkg/bindinfo/binding_plan_generation_test.go +++ b/pkg/bindinfo/binding_plan_generation_test.go @@ -102,10 +102,12 @@ func TestStartState(t *testing.T) { vardef.TiDBOptRiskRangeSkewRatio, vardef.TiDBOptRiskGroupNDVSkewRatio, vardef.TiDBOptSelectivityFactor, + vardef.TiDBOptPreferRangeScan, + vardef.TiDBOptEnableNoDecorrelateInSelect, } fixes := []uint64{fixcontrol.Fix44855, fixcontrol.Fix45132, fixcontrol.Fix52869} state, err := getStartState(vars, fixes) require.NoError(t, err) - require.Equal(t, state.Encode(), "1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,0.0100,0.0000,0.0000,0.0000,0.8000,OFF,1000,OFF") + require.Equal(t, state.Encode(), "1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,0.0100,0.0000,0.0000,0.0000,0.8000,true,false,OFF,1000,OFF") } diff --git a/pkg/planner/core/expression_rewriter.go b/pkg/planner/core/expression_rewriter.go index 8fb3628718cfa..062eb1cb1bb5c 100644 --- a/pkg/planner/core/expression_rewriter.go +++ b/pkg/planner/core/expression_rewriter.go @@ -257,7 +257,7 @@ func (b *PlanBuilder) getExpressionRewriter(ctx context.Context, p base.LogicalP if len(b.rewriterPool) < b.rewriterCounter { rewriter = &expressionRewriter{ sctx: b.ctx.GetExprCtx(), ctx: ctx, - planCtx: &exprRewriterPlanCtx{plan: p, builder: b, rollExpand: b.currentBlockExpand}, + planCtx: &exprRewriterPlanCtx{plan: p, builder: b, curClause: b.curClause, rollExpand: b.currentBlockExpand}, } b.rewriterPool = append(b.rewriterPool, rewriter) return @@ -273,6 +273,7 @@ func (b *PlanBuilder) getExpressionRewriter(ctx context.Context, p base.LogicalP rewriter.ctx = ctx rewriter.err = nil rewriter.planCtx.plan = p + rewriter.planCtx.curClause = b.curClause rewriter.planCtx.aggrMap = nil rewriter.planCtx.insertPlan = nil rewriter.planCtx.rollExpand = b.currentBlockExpand @@ -330,6 +331,9 @@ type exprRewriterPlanCtx struct { plan base.LogicalPlan builder *PlanBuilder + // curClause tracks which part of the query is being processed + curClause clauseCode + aggrMap map[*ast.AggregateFuncExpr]int windowMap map[*ast.WindowFuncExpr]int @@ -735,7 +739,7 @@ func (er *expressionRewriter) handleCompareSubquery(ctx context.Context, planCtx return v, true } corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, handlingCompareSubquery) // Only (a,b,c) = any (...) and (a,b,c) != all (...) can use row expression. canMultiCol := (!v.All && v.Op == opcode.EQ) || (v.All && v.Op == opcode.NE) @@ -1041,7 +1045,8 @@ func (er *expressionRewriter) handleExistSubquery(ctx context.Context, planCtx * } np = er.popExistsSubPlan(planCtx, np) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, handlingExistsSubquery) + semiJoinRewrite := hintFlags&hint.HintFlagSemiJoinRewrite > 0 if semiJoinRewrite && noDecorrelate { b.ctx.GetSessionVars().StmtCtx.SetHintWarning( @@ -1212,7 +1217,7 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp lt, rt := lexpr.GetType(er.sctx.GetEvalCtx()), rexpr.GetType(er.sctx.GetEvalCtx()) collFlag := collate.CompatibleCollate(lt.GetCollate(), rt.GetCollate()) corCols := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags) + noDecorrelate := isNoDecorrelate(planCtx, corCols, hintFlags, handlingInSubquery) // If it's not the form of `not in (SUBQUERY)`, // and has no correlated column from the current level plan(if the correlated column is from upper level, @@ -1264,12 +1269,30 @@ func (er *expressionRewriter) handleInSubquery(ctx context.Context, planCtx *exp return v, true } -func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64) bool { +func isNoDecorrelate(planCtx *exprRewriterPlanCtx, corCols []*expression.CorrelatedColumn, hintFlags uint64, sCtx subQueryCtx) bool { noDecorrelate := hintFlags&hint.HintFlagNoDecorrelate > 0 - if noDecorrelate && len(corCols) == 0 { - planCtx.builder.ctx.GetSessionVars().StmtCtx.SetHintWarning( - "NO_DECORRELATE() is inapplicable because there are no correlated columns.") - noDecorrelate = false + + if len(corCols) == 0 { + if noDecorrelate { + planCtx.builder.ctx.GetSessionVars().StmtCtx.SetHintWarning( + "NO_DECORRELATE() is inapplicable because there are no correlated columns.") + noDecorrelate = false + } + } else { + semiJoinRewrite := hintFlags&hint.HintFlagSemiJoinRewrite > 0 + // We can't override noDecorrelate via the variable for EXISTS subqueries with semi join rewrite + // as this will cause a conflict that will result in both being disabled in later code + if !(semiJoinRewrite && sCtx == handlingExistsSubquery) { + // Only support scalar and exists subqueries + validSubqType := sCtx == handlingScalarSubquery || sCtx == handlingExistsSubquery + if validSubqType && planCtx.curClause == fieldList { // subquery is in the select list + planCtx.builder.ctx.GetSessionVars().RecordRelevantOptVar(vardef.TiDBOptEnableNoDecorrelateInSelect) + // If it isn't already enabled via hint, and variable is set, then enable it + if !noDecorrelate && planCtx.builder.ctx.GetSessionVars().EnableNoDecorrelateInSelect { + noDecorrelate = true + } + } + } } return noDecorrelate } @@ -1285,7 +1308,7 @@ func (er *expressionRewriter) handleScalarSubquery(ctx context.Context, planCtx } np = planCtx.builder.buildMaxOneRow(np) correlatedColumn := coreusage.ExtractCorColumnsBySchema4LogicalPlan(np, planCtx.plan.Schema()) - noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags) + noDecorrelate := isNoDecorrelate(planCtx, correlatedColumn, hintFlags, handlingScalarSubquery) if planCtx.builder.disableSubQueryPreprocessing || len(coreusage.ExtractCorrelatedCols4LogicalPlan(np)) > 0 || hasCTEConsumerInSubPlan(np) { planCtx.plan = planCtx.builder.buildApplyWithJoinType(planCtx.plan, np, logicalop.LeftOuterJoin, noDecorrelate) diff --git a/pkg/planner/core/logical_plan_builder.go b/pkg/planner/core/logical_plan_builder.go index 7a207d8062077..5e47a8a9858be 100644 --- a/pkg/planner/core/logical_plan_builder.go +++ b/pkg/planner/core/logical_plan_builder.go @@ -1992,7 +1992,7 @@ func (b *PlanBuilder) checkOrderByInDistinct(byItem *ast.ByItem, idx int, expr e // Check if referenced columns of expressions in ORDER BY whole match some fields in DISTINCT, // both original expression and alias can be referenced. // e.g. - // select distinct a from t order by sin(a); ✔ + // select distinct sin(a) from t order by a; ✔ // select distinct a, b from t order by a+b; ✔ // select distinct count(a), sum(a) from t group by b order by sum(a); ✔ cols := expression.ExtractColumns(expr) @@ -2546,6 +2546,10 @@ func (b *PlanBuilder) extractCorrelatedAggFuncs(ctx context.Context, p base.Logi corCols = append(corCols, expression.ExtractCorColumns(expr)...) cols = append(cols, expression.ExtractColumns(expr)...) } + // If decorrelation is disabled, don't extract correlated aggregates + if b.noDecorrelate && len(corCols) > 0 { + continue + } if len(corCols) > 0 && len(cols) == 0 { outer = append(outer, agg) } @@ -2610,6 +2614,9 @@ type correlatedAggregateResolver struct { // correlatedAggFuncs stores aggregate functions which belong to outer query correlatedAggFuncs []*ast.AggregateFuncExpr + + // noDecorrelate indicates whether decorrelation should be disabled for this resolver + noDecorrelate bool } // Enter implements Visitor interface. @@ -2784,9 +2791,10 @@ func (r *correlatedAggregateResolver) Leave(n ast.Node) (ast.Node, bool) { // in the outer query from all the sub-queries inside SELECT fields. func (b *PlanBuilder) resolveCorrelatedAggregates(ctx context.Context, sel *ast.SelectStmt, p base.LogicalPlan) (map[*ast.AggregateFuncExpr]int, error) { resolver := &correlatedAggregateResolver{ - ctx: ctx, - b: b, - outerPlan: p, + ctx: ctx, + b: b, + outerPlan: p, + noDecorrelate: b.noDecorrelate, } correlatedAggList := make([]*ast.AggregateFuncExpr, 0) for _, field := range sel.Fields.Fields { diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index 9f762799f3c77..820769948bf5e 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -303,6 +303,9 @@ type PlanBuilder struct { // disableSubQueryPreprocessing indicates whether to pre-process uncorrelated sub-queries in rewriting stage. disableSubQueryPreprocessing bool + // noDecorrelate indicates whether decorrelation should be disabled for correlated aggregates in subqueries + noDecorrelate bool + // allowBuildCastArray indicates whether allow cast(... as ... array). allowBuildCastArray bool // resolveCtx is set when calling Build, it's only effective in the current Build call. @@ -462,6 +465,7 @@ func (b *PlanBuilder) Init(sctx base.PlanContext, is infoschema.InfoSchema, proc b.is = is b.hintProcessor = processor b.isForUpdateRead = sctx.GetSessionVars().IsPessimisticReadConsistency() + b.noDecorrelate = sctx.GetSessionVars().EnableNoDecorrelateInSelect if savedBlockNames == nil { return b, nil } @@ -494,6 +498,7 @@ func (b *PlanBuilder) ResetForReuse() *PlanBuilder { b.colMapper = saveColMapper b.handleHelper = saveHandleHelper b.correlatedAggMapper = saveCorrelateAggMapper + b.noDecorrelate = false // Add more fields if they are safe to be reused. diff --git a/pkg/sessionctx/vardef/tidb_vars.go b/pkg/sessionctx/vardef/tidb_vars.go index c7a1d5cfe682d..f33a9f2e3cc3a 100644 --- a/pkg/sessionctx/vardef/tidb_vars.go +++ b/pkg/sessionctx/vardef/tidb_vars.go @@ -312,6 +312,9 @@ const ( // TiDBOptPreferRangeScan is used to enable/disable the optimizer to always prefer range scan over table scan, ignoring their costs. TiDBOptPreferRangeScan = "tidb_opt_prefer_range_scan" + // TiDBOptEnableNoDecorrelateInSelect is used to control whether to enable the NO_DECORRELATE hint for subqueries in the select list. + TiDBOptEnableNoDecorrelateInSelect = "tidb_opt_enable_no_decorrelate_in_select" + // TiDBOptEnableCorrelationAdjustment is used to indicates if enable correlation adjustment. TiDBOptEnableCorrelationAdjustment = "tidb_opt_enable_correlation_adjustment" @@ -1385,6 +1388,7 @@ const ( DefOptForceInlineCTE = false DefOptInSubqToJoinAndAgg = true DefOptPreferRangeScan = true + DefOptEnableNoDecorrelateInSelect = false DefBatchInsert = false DefBatchDelete = false DefBatchCommit = false diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index 6c2f4446d9a7c..ea010af77201d 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -1137,6 +1137,9 @@ type SessionVars struct { // EnablePipelinedWindowExec enables executing window functions in a pipelined manner. EnablePipelinedWindowExec bool + // EnableNoDecorrelateInSelect enables the NO_DECORRELATE hint for subqueries in the select list. + EnableNoDecorrelateInSelect bool + // AllowProjectionPushDown enables pushdown projection on TiKV. AllowProjectionPushDown bool @@ -2203,6 +2206,7 @@ func NewSessionVars(hctx HookContext) *SessionVars { RiskScaleNDVSkewRatio: vardef.DefOptRiskScaleNDVSkewRatio, RiskGroupNDVSkewRatio: vardef.DefOptRiskGroupNDVSkewRatio, EnableOuterJoinReorder: vardef.DefTiDBEnableOuterJoinReorder, + EnableNoDecorrelateInSelect: vardef.DefOptEnableNoDecorrelateInSelect, RetryLimit: vardef.DefTiDBRetryLimit, DisableTxnAutoRetry: vardef.DefTiDBDisableTxnAutoRetry, DDLReorgPriority: kv.PriorityLow, diff --git a/pkg/sessionctx/variable/setvar_affect.go b/pkg/sessionctx/variable/setvar_affect.go index 2daeb1960977f..a2118288159db 100644 --- a/pkg/sessionctx/variable/setvar_affect.go +++ b/pkg/sessionctx/variable/setvar_affect.go @@ -116,6 +116,7 @@ var isHintUpdatableVerified = map[string]struct{}{ "tidb_store_batch_size": {}, "mpp_version": {}, "tidb_enable_inl_join_inner_multi_pattern": {}, + "tidb_opt_enable_no_decorrelate_in_select": {}, "tidb_opt_enable_late_materialization": {}, "tidb_opt_ordering_index_selectivity_threshold": {}, "tidb_opt_ordering_index_selectivity_ratio": {}, diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index 4dc83169957d8..2ab211c34d7cc 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -2364,6 +2364,10 @@ var defaultSysVars = []*SysVar{ s.EnablePipelinedWindowExec = TiDBOptOn(val) return nil }}, + {Scope: vardef.ScopeGlobal | vardef.ScopeSession, Name: vardef.TiDBOptEnableNoDecorrelateInSelect, Value: BoolToOnOff(vardef.DefOptEnableNoDecorrelateInSelect), Type: vardef.TypeBool, SetSession: func(s *SessionVars, val string) error { + s.EnableNoDecorrelateInSelect = TiDBOptOn(val) + return nil + }}, {Scope: vardef.ScopeGlobal | vardef.ScopeSession, Name: vardef.TiDBEnableStrictDoubleTypeCheck, Value: BoolToOnOff(vardef.DefEnableStrictDoubleTypeCheck), Type: vardef.TypeBool, SetSession: func(s *SessionVars, val string) error { s.EnableStrictDoubleTypeCheck = TiDBOptOn(val) return nil