Skip to content

planner: record explored plans into tidb_statement_stats when running explain explore #61850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ti-chi-bot merged 7 commits into from
Jun 23, 2025
Merged

planner: record explored plans into tidb_statement_stats when running explain explore #61850

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
adb5943
fixup
qw4990 Jun 19, 2025
f77d2de
fixup
qw4990 Jun 20, 2025
007cefb
fixup
qw4990 Jun 20, 2025
a27f460
fixup
qw4990 Jun 20, 2025
f29e3cb
Merge remote-tracking branch 'upstream/master' into explain-explore-s…
qw4990 Jun 23, 2025
aa11ec5
fixup
qw4990 Jun 23, 2025
4636a70
fixup
qw4990 Jun 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkg/bindinfo/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ go_test(
embed = [":bindinfo"],
flaky = True,
race = "on",
shard_count = 42,
shard_count = 43,
deps = [
"//pkg/parser",
"//pkg/parser/ast",
Expand Down
41 changes: 41 additions & 0 deletions pkg/bindinfo/binding_auto.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,10 @@ func (ba *bindingAuto) ExplorePlansForSQL(currentDB, sqlOrDigest, charset, colla
if err != nil {
return nil, err
}
generatedPlans, err = ba.recordIntoStmtStats(generatedPlans)
if err != nil {
return nil, err
}

if analyze {
if err := ba.runToGetExecInfo(generatedPlans); err != nil {
Expand All @@ -117,6 +121,43 @@ func (ba *bindingAuto) ExplorePlansForSQL(currentDB, sqlOrDigest, charset, colla
return planCandidates, err
}

// recordIntoStmtStats records these plans into information_schema.tidb_statements_stats table for later usage.
func (ba *bindingAuto) recordIntoStmtStats(plans []*BindingPlanInfo) (reproduciblePlans []*BindingPlanInfo, err error) {
reproduciblePlans = make([]*BindingPlanInfo, 0, len(plans))
for _, plan := range plans {
if err := callWithSCtx(ba.sPool, false, func(sctx sessionctx.Context) error {
vars := sctx.GetSessionVars()
defer func(db string, usePlanBaselines, inExplainExplore bool) {
vars.CurrentDB = db
vars.UsePlanBaselines = usePlanBaselines
vars.InExplainExplore = inExplainExplore
}(vars.CurrentDB, vars.UsePlanBaselines, vars.InExplainExplore)
vars.CurrentDB = plan.Binding.Db
vars.UsePlanBaselines = false
vars.InExplainExplore = true
_, _, err := execRows(sctx, plan.BindSQL)
if err != nil {
return err
}

execInfo, err := ba.getPlanExecInfo(plan.Binding.PlanDigest)
if err != nil {
return err
}
// Due to the flaw of `core.GenHintsFromFlatPlan`, sometimes we might not be able to reproduce the prior
// plan exactly with `plan.Binding.Hint`.
// In this case we can't get any record in `tidb_statements_stats` via its PlanDigest, and since
if execInfo != nil {
reproduciblePlans = append(reproduciblePlans, plan)
}
return nil
}); err != nil {
return nil, err
}
}
return reproduciblePlans, nil
}

// runToGetExecInfo runs these plans to get their execution info.
func (ba *bindingAuto) runToGetExecInfo(plans []*BindingPlanInfo) error {
// TODO: support setting timeout
Expand Down
10 changes: 10 additions & 0 deletions pkg/bindinfo/binding_auto_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,16 @@ func TestRelevantOptVarsAndFixes(t *testing.T) {
}
}

func TestExplainExploreInStmtStats(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
tk.MustExec("use test")
tk.MustExec(`create table t (a int, b int, key(a))`)
tk.MustQuery(`explain explore select count(1) from t where a=1`)
rs := tk.MustQuery("select digest_text from information_schema.tidb_statements_stats where digest_text = 'select count ( ? ) from `t` where `a` = ?'").Rows()
require.True(t, len(rs) > 0)
}

func TestExplainExploreAnalyze(t *testing.T) {
store := testkit.CreateMockStore(t)
tk := testkit.NewTestKit(t, store)
Expand Down
1 change: 0 additions & 1 deletion pkg/bindinfo/testdata/binding_auto_suite_in.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
{
"name": "TestPlanGeneration",
"cases": [
"explain explore select count(1) from t",
"explain explore select max(b) from t where a=1",
"explain explore select * from t where a=1",
"explain explore select * from t where b=1",
Expand Down
136 changes: 0 additions & 136 deletions pkg/bindinfo/testdata/binding_auto_suite_out.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,55 +2,6 @@
{
"Name": "TestPlanGeneration",
"Cases": [
{
"SQL": "explain explore select count(1) from t",
"Plan": [
[
"HashAgg 1.00 root funcs:count(Column#7)->Column#5",
"└─IndexReader 1.00 root index:HashAgg",
" └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#7",
" └─IndexFullScan 10000.00 cop[tikv] table:t, index:a(a) keep order:false, stats:pseudo"
],
[
"HashAgg 1.00 root funcs:count(1)->Column#5",
"└─IndexReader 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t, index:a(a) keep order:false, stats:pseudo"
],
[
"StreamAgg 1.00 root funcs:count(Column#10)->Column#5",
"└─IndexReader 1.00 root index:StreamAgg",
" └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#10",
" └─IndexFullScan 10000.00 cop[tikv] table:t, index:a(a) keep order:false, stats:pseudo"
],
[
"StreamAgg 1.00 root funcs:count(1)->Column#5",
"└─TableReader 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo"
],
[
"StreamAgg 1.00 root funcs:count(Column#10)->Column#5",
"└─TableReader 1.00 root data:StreamAgg",
" └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#10",
" └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo"
],
[
"HashAgg 1.00 root funcs:count(1)->Column#5",
"└─TableReader 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo"
],
[
"StreamAgg 1.00 root funcs:count(1)->Column#5",
"└─IndexReader 10000.00 root index:IndexFullScan",
" └─IndexFullScan 10000.00 cop[tikv] table:t, index:a(a) keep order:false, stats:pseudo"
],
[
"HashAgg 1.00 root funcs:count(Column#7)->Column#5",
"└─TableReader 1.00 root data:HashAgg",
" └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#7",
" └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo"
]
]
},
{
"SQL": "explain explore select max(b) from t where a=1",
"Plan": [
Expand Down Expand Up @@ -118,39 +69,6 @@
{
"SQL": "explain explore select * from t1, t2, t3",
"Plan": [
[
Copy link
Contributor Author

@qw4990 qw4990 Jun 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Expected, can't reproduce these plans through hints.

"Projection 1000000000000.00 root test.t1.a, test.t1.b, test.t1.c, test.t2.a, test.t2.b, test.t2.c, test.t3.a, test.t3.b, test.t3.c",
"└─HashJoin 1000000000000.00 root CARTESIAN inner join",
" ├─HashJoin(Probe) 100000000.00 root CARTESIAN inner join",
" │ ├─TableReader(Probe) 10000.00 root data:TableFullScan",
" │ │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo",
" │ └─TableReader(Build) 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo",
" └─TableReader(Build) 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
[
"Projection 1000000000000.00 root test.t1.a, test.t1.b, test.t1.c, test.t2.a, test.t2.b, test.t2.c, test.t3.a, test.t3.b, test.t3.c",
"└─HashJoin 1000000000000.00 root CARTESIAN inner join",
" ├─HashJoin(Probe) 100000000.00 root CARTESIAN inner join",
" │ ├─TableReader(Probe) 10000.00 root data:TableFullScan",
" │ │ └─TableFullScan 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo",
" │ └─TableReader(Build) 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo",
" └─TableReader(Build) 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
[
"Projection 1000000000000.00 root test.t1.a, test.t1.b, test.t1.c, test.t2.a, test.t2.b, test.t2.c, test.t3.a, test.t3.b, test.t3.c",
"└─HashJoin 1000000000000.00 root CARTESIAN inner join",
" ├─HashJoin(Probe) 100000000.00 root CARTESIAN inner join",
" │ ├─TableReader(Probe) 10000.00 root data:TableFullScan",
" │ │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" │ └─TableReader(Build) 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo",
" └─TableReader(Build) 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo"
],
[
"HashJoin 1000000000000.00 root CARTESIAN inner join",
"├─HashJoin(Probe) 100000000.00 root CARTESIAN inner join",
Expand All @@ -160,45 +78,12 @@
"│ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo",
"└─TableReader(Build) 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo"
],
[
"Projection 1000000000000.00 root test.t1.a, test.t1.b, test.t1.c, test.t2.a, test.t2.b, test.t2.c, test.t3.a, test.t3.b, test.t3.c",
"└─HashJoin 1000000000000.00 root CARTESIAN inner join",
" ├─HashJoin(Probe) 100000000.00 root CARTESIAN inner join",
" │ ├─TableReader(Probe) 10000.00 root data:TableFullScan",
" │ │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo",
" │ └─TableReader(Build) 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" └─TableReader(Build) 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo"
],
[
"Projection 1000000000000.00 root test.t1.a, test.t1.b, test.t1.c, test.t2.a, test.t2.b, test.t2.c, test.t3.a, test.t3.b, test.t3.c",
"└─HashJoin 1000000000000.00 root CARTESIAN inner join",
" ├─HashJoin(Probe) 100000000.00 root CARTESIAN inner join",
" │ ├─TableReader(Probe) 10000.00 root data:TableFullScan",
" │ │ └─TableFullScan 10000.00 cop[tikv] table:t3 keep order:false, stats:pseudo",
" │ └─TableReader(Build) 10000.00 root data:TableFullScan",
" │ └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo",
" └─TableReader(Build) 10000.00 root data:TableFullScan",
" └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo"
]
]
},
{
"SQL": "explain explore select * from t1, t2 where t1.a=t2.a",
"Plan": [
[
"Projection 12487.50 root test.t1.a, test.t1.b, test.t1.c, test.t2.a, test.t2.b, test.t2.c",
"└─IndexHashJoin 12487.50 root inner join, inner:IndexLookUp, outer key:test.t2.a, inner key:test.t1.a, equal cond:eq(test.t2.a, test.t1.a)",
" ├─IndexLookUp(Build) 9990.00 root ",
" │ ├─IndexFullScan(Build) 9990.00 cop[tikv] table:t2, index:a(a) keep order:false, stats:pseudo",
" │ └─TableRowIDScan(Probe) 9990.00 cop[tikv] table:t2 keep order:false, stats:pseudo",
" └─IndexLookUp(Probe) 12487.50 root ",
" ├─Selection(Build) 12487.50 cop[tikv] not(isnull(test.t1.a))",
" │ └─IndexRangeScan 12500.00 cop[tikv] table:t1, index:a(a) range: decided by [eq(test.t1.a, test.t2.a)], keep order:false, stats:pseudo",
" └─TableRowIDScan(Probe) 12487.50 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
[
"MergeJoin 12487.50 root inner join, left key:test.t1.a, right key:test.t2.a",
"├─Projection(Probe) 9990.00 root test.t1.a, test.t1.b, test.t1.c",
Expand All @@ -220,27 +105,6 @@
" │ └─IndexRangeScan 12500.00 cop[tikv] table:t2, index:a(a) range: decided by [eq(test.t2.a, test.t1.a)], keep order:false, stats:pseudo",
" └─TableRowIDScan(Probe) 12487.50 cop[tikv] table:t2 keep order:false, stats:pseudo"
],
[
"Projection 12487.50 root test.t1.a, test.t1.b, test.t1.c, test.t2.a, test.t2.b, test.t2.c",
"└─IndexHashJoin 12487.50 root inner join, inner:IndexLookUp, outer key:test.t2.a, inner key:test.t1.a, equal cond:eq(test.t2.a, test.t1.a)",
" ├─TableReader(Build) 9990.00 root data:Selection",
" │ └─Selection 9990.00 cop[tikv] not(isnull(test.t2.a))",
" │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo",
" └─IndexLookUp(Probe) 12487.50 root ",
" ├─Selection(Build) 12487.50 cop[tikv] not(isnull(test.t1.a))",
" │ └─IndexRangeScan 12500.00 cop[tikv] table:t1, index:a(a) range: decided by [eq(test.t1.a, test.t2.a)], keep order:false, stats:pseudo",
" └─TableRowIDScan(Probe) 12487.50 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
[
"Projection 12487.50 root test.t1.a, test.t1.b, test.t1.c, test.t2.a, test.t2.b, test.t2.c",
"└─HashJoin 12487.50 root inner join, equal:[eq(test.t2.a, test.t1.a)]",
" ├─TableReader(Probe) 9990.00 root data:Selection",
" │ └─Selection 9990.00 cop[tikv] not(isnull(test.t2.a))",
" │ └─TableFullScan 10000.00 cop[tikv] table:t2 keep order:false, stats:pseudo",
" └─TableReader(Build) 9990.00 root data:Selection",
" └─Selection 9990.00 cop[tikv] not(isnull(test.t1.a))",
" └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo"
],
[
"HashJoin 12487.50 root inner join, equal:[eq(test.t1.a, test.t2.a)]",
"├─TableReader(Probe) 9990.00 root data:Selection",
Expand Down
6 changes: 4 additions & 2 deletions pkg/executor/adapter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1939,7 +1939,9 @@ func (a *ExecStmt) SummaryStmt(succ bool) {
}

// Internal SQLs must also be recorded to keep the consistency of `PrevStmt` and `PrevStmtDigest`.
if !stmtsummaryv2.Enabled() || ((sessVars.InRestrictedSQL || len(userString) == 0) && !stmtsummaryv2.EnabledInternal()) {
// If this SQL is under `explain explore {SQL}`, we still want to record them in stmt summary.
isInternalSQL := (sessVars.InRestrictedSQL || len(userString) == 0) && !sessVars.InExplainExplore
if !stmtsummaryv2.Enabled() || (isInternalSQL && !stmtsummaryv2.EnabledInternal()) {
sessVars.SetPrevStmtDigest("")
return
}
Expand Down Expand Up @@ -2037,7 +2039,7 @@ func (a *ExecStmt) SummaryStmt(succ bool) {
stmtExecInfo.MemMax = memMax
stmtExecInfo.DiskMax = diskMax
stmtExecInfo.StartTime = sessVars.StartTime
stmtExecInfo.IsInternal = sessVars.InRestrictedSQL
stmtExecInfo.IsInternal = isInternalSQL
stmtExecInfo.Succeed = succ
stmtExecInfo.PlanInCache = sessVars.FoundInPlanCache
stmtExecInfo.PlanInBinding = sessVars.FoundInBinding
Expand Down
3 changes: 3 additions & 0 deletions pkg/sessionctx/variable/session.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -863,6 +863,9 @@ type SessionVars struct {
// InRestrictedSQL indicates if the session is handling restricted SQL execution.
InRestrictedSQL bool

// InExplainExplore indicates if this statement is under EXPLAIN EXPLORE.
InExplainExplore bool

// SnapshotTS is used for reading history data. For simplicity, SnapshotTS only supports distsql request.
SnapshotTS uint64

Expand Down