From 47279f5a12d5918105de377ecba70e7c3fba3bce Mon Sep 17 00:00:00 2001 From: Ti Chi Robot Date: Mon, 26 Aug 2024 00:06:14 +0800 Subject: [PATCH] statistics: add metrics for unneeded analyze table (#54822) (#55577) close pingcap/tidb#54823 --- pkg/executor/test/analyzetest/BUILD.bazel | 1 - pkg/executor/test/analyzetest/analyze_test.go | 27 +++++++------- .../analyzetest/memorycontrol/BUILD.bazel | 2 +- .../memorycontrol/memory_control_test.go | 8 ++--- pkg/statistics/BUILD.bazel | 1 - .../handle/autoanalyze/autoanalyze.go | 7 ++-- .../handle/autoanalyze/autoanalyze_test.go | 25 +++++++------ pkg/statistics/handle/cache/statscache.go | 9 ++++- pkg/statistics/handle/metrics/metrics.go | 1 + pkg/statistics/handle/updatetest/BUILD.bazel | 1 - .../handle/updatetest/update_test.go | 35 +++++++++---------- pkg/statistics/integration_test.go | 5 ++- pkg/statistics/table.go | 17 +++++++++ pkg/ttl/ttlworker/BUILD.bazel | 2 +- .../ttlworker/job_manager_integration_test.go | 14 ++++---- 15 files changed, 85 insertions(+), 70 deletions(-) diff --git a/pkg/executor/test/analyzetest/BUILD.bazel b/pkg/executor/test/analyzetest/BUILD.bazel index f3504f59f6e80..f8980a6d33af0 100644 --- a/pkg/executor/test/analyzetest/BUILD.bazel +++ b/pkg/executor/test/analyzetest/BUILD.bazel @@ -26,7 +26,6 @@ go_test( "//pkg/sessionctx", "//pkg/sessionctx/variable", "//pkg/statistics", - "//pkg/statistics/handle/autoanalyze", "//pkg/testkit", "//pkg/util/dbterror/exeerrors", "@com_github_pingcap_errors//:errors", diff --git a/pkg/executor/test/analyzetest/analyze_test.go b/pkg/executor/test/analyzetest/analyze_test.go index 59de964d96ace..a20c2f493a380 100644 --- a/pkg/executor/test/analyzetest/analyze_test.go +++ b/pkg/executor/test/analyzetest/analyze_test.go @@ -39,7 +39,6 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/statistics" - "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" "github.com/stretchr/testify/require" @@ -743,11 +742,11 @@ func TestSavedAnalyzeOptions(t *testing.T) { tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal2)) }() tk.MustExec("set global tidb_auto_analyze_ratio = 0.01") - originalVal3 := autoanalyze.AutoAnalyzeMinCnt + originalVal3 := statistics.AutoAnalyzeMinCnt defer func() { - autoanalyze.AutoAnalyzeMinCnt = originalVal3 + statistics.AutoAnalyzeMinCnt = originalVal3 }() - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 tk.MustExec("use test") tk.MustExec("set @@session.tidb_analyze_version = 2") @@ -1085,11 +1084,11 @@ func TestSavedAnalyzeColumnOptions(t *testing.T) { tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal2)) }() tk.MustExec("set global tidb_auto_analyze_ratio = 0.01") - originalVal3 := autoanalyze.AutoAnalyzeMinCnt + originalVal3 := statistics.AutoAnalyzeMinCnt defer func() { - autoanalyze.AutoAnalyzeMinCnt = originalVal3 + statistics.AutoAnalyzeMinCnt = originalVal3 }() - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 originalVal4 := tk.MustQuery("select @@tidb_enable_column_tracking").Rows()[0][0].(string) defer func() { tk.MustExec(fmt.Sprintf("set global tidb_enable_column_tracking = %v", originalVal4)) @@ -1966,9 +1965,9 @@ func testKillAutoAnalyze(t *testing.T, ver int) { tk := testkit.NewTestKit(t, store) oriStart := tk.MustQuery("select @@tidb_auto_analyze_start_time").Rows()[0][0].(string) oriEnd := tk.MustQuery("select @@tidb_auto_analyze_end_time").Rows()[0][0].(string) - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 defer func() { - autoanalyze.AutoAnalyzeMinCnt = 1000 + statistics.AutoAnalyzeMinCnt = 1000 tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_start_time='%v'", oriStart)) tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_end_time='%v'", oriEnd)) }() @@ -2050,9 +2049,9 @@ func TestKillAutoAnalyzeIndex(t *testing.T) { tk := testkit.NewTestKit(t, store) oriStart := tk.MustQuery("select @@tidb_auto_analyze_start_time").Rows()[0][0].(string) oriEnd := tk.MustQuery("select @@tidb_auto_analyze_end_time").Rows()[0][0].(string) - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 defer func() { - autoanalyze.AutoAnalyzeMinCnt = 1000 + statistics.AutoAnalyzeMinCnt = 1000 tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_start_time='%v'", oriStart)) tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_end_time='%v'", oriEnd)) }() @@ -2805,12 +2804,12 @@ func TestAutoAnalyzeAwareGlobalVariableChange(t *testing.T) { "3 0", )) - originalVal1 := autoanalyze.AutoAnalyzeMinCnt + originalVal1 := statistics.AutoAnalyzeMinCnt originalVal2 := tk.MustQuery("select @@global.tidb_auto_analyze_ratio").Rows()[0][0].(string) - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 tk.MustExec("set global tidb_auto_analyze_ratio = 0.001") defer func() { - autoanalyze.AutoAnalyzeMinCnt = originalVal1 + statistics.AutoAnalyzeMinCnt = originalVal1 tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal2)) }() diff --git a/pkg/executor/test/analyzetest/memorycontrol/BUILD.bazel b/pkg/executor/test/analyzetest/memorycontrol/BUILD.bazel index bf8c263bb6016..e45a5b4baf634 100644 --- a/pkg/executor/test/analyzetest/memorycontrol/BUILD.bazel +++ b/pkg/executor/test/analyzetest/memorycontrol/BUILD.bazel @@ -13,7 +13,7 @@ go_test( "//pkg/config", "//pkg/executor", "//pkg/sessionctx/variable", - "//pkg/statistics/handle/autoanalyze", + "//pkg/statistics", "//pkg/testkit", "//pkg/util", "//pkg/util/memory", diff --git a/pkg/executor/test/analyzetest/memorycontrol/memory_control_test.go b/pkg/executor/test/analyzetest/memorycontrol/memory_control_test.go index a64e517e60d95..fcd2b409a382c 100644 --- a/pkg/executor/test/analyzetest/memorycontrol/memory_control_test.go +++ b/pkg/executor/test/analyzetest/memorycontrol/memory_control_test.go @@ -22,7 +22,7 @@ import ( "github.com/pingcap/failpoint" "github.com/pingcap/tidb/pkg/executor" - "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze" + "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/util" "github.com/pingcap/tidb/pkg/util/memory" @@ -144,12 +144,12 @@ func TestGlobalMemoryControlForAutoAnalyze(t *testing.T) { require.Len(t, rs0.Rows(), 0) h := dom.StatsHandle() - originalVal4 := autoanalyze.AutoAnalyzeMinCnt + originalVal4 := statistics.AutoAnalyzeMinCnt originalVal5 := tk.MustQuery("select @@global.tidb_auto_analyze_ratio").Rows()[0][0].(string) - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 tk.MustExec("set global tidb_auto_analyze_ratio = 0.001") defer func() { - autoanalyze.AutoAnalyzeMinCnt = originalVal4 + statistics.AutoAnalyzeMinCnt = originalVal4 tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_ratio = %v", originalVal5)) }() diff --git a/pkg/statistics/BUILD.bazel b/pkg/statistics/BUILD.bazel index 20ee8a66d1270..dd252b6ddfc4c 100644 --- a/pkg/statistics/BUILD.bazel +++ b/pkg/statistics/BUILD.bazel @@ -86,7 +86,6 @@ go_test( "//pkg/parser/mysql", "//pkg/sessionctx", "//pkg/sessionctx/stmtctx", - "//pkg/statistics/handle/autoanalyze", "//pkg/testkit", "//pkg/testkit/testdata", "//pkg/testkit/testmain", diff --git a/pkg/statistics/handle/autoanalyze/autoanalyze.go b/pkg/statistics/handle/autoanalyze/autoanalyze.go index 29c0c1f9af1c7..5e7eb96635dea 100644 --- a/pkg/statistics/handle/autoanalyze/autoanalyze.go +++ b/pkg/statistics/handle/autoanalyze/autoanalyze.go @@ -274,14 +274,11 @@ func RandomPickOneTableAndTryAutoAnalyze( return false } -// AutoAnalyzeMinCnt means if the count of table is less than this value, we needn't do auto analyze. -var AutoAnalyzeMinCnt int64 = 1000 - func autoAnalyzeTable(sctx sessionctx.Context, statsHandle statsutil.StatsHandle, tblInfo *model.TableInfo, statsTbl *statistics.Table, ratio float64, sql string, params ...interface{}) bool { - if statsTbl.Pseudo || statsTbl.RealtimeCount < AutoAnalyzeMinCnt { + if statsTbl.Pseudo || statsTbl.RealtimeCount < statistics.AutoAnalyzeMinCnt { return false } if needAnalyze, reason := NeedAnalyzeTable(statsTbl, 20*statsHandle.Lease(), ratio); needAnalyze { @@ -375,7 +372,7 @@ func autoAnalyzePartitionTableInDynamicMode(sctx sessionctx.Context, partitionNames := make([]interface{}, 0, len(partitionDefs)) for _, def := range partitionDefs { partitionStatsTbl := statsHandle.GetPartitionStats(tblInfo, def.ID) - if partitionStatsTbl.Pseudo || partitionStatsTbl.RealtimeCount < AutoAnalyzeMinCnt { + if partitionStatsTbl.Pseudo || partitionStatsTbl.RealtimeCount < statistics.AutoAnalyzeMinCnt { continue } if needAnalyze, reason := NeedAnalyzeTable(partitionStatsTbl, 20*statsHandle.Lease(), ratio); needAnalyze { diff --git a/pkg/statistics/handle/autoanalyze/autoanalyze_test.go b/pkg/statistics/handle/autoanalyze/autoanalyze_test.go index b090c416c86fb..c04fbac5ab87a 100644 --- a/pkg/statistics/handle/autoanalyze/autoanalyze_test.go +++ b/pkg/statistics/handle/autoanalyze/autoanalyze_test.go @@ -44,9 +44,9 @@ func TestAutoAnalyzeLockedTable(t *testing.T) { tk.MustExec("lock stats t") is := dom.InfoSchema() require.NoError(t, h.Update(is)) - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 defer func() { - autoanalyze.AutoAnalyzeMinCnt = 1000 + statistics.AutoAnalyzeMinCnt = 1000 }() // Try to analyze the locked table, it should not analyze the table. require.False(t, dom.StatsHandle().HandleAutoAnalyze(dom.InfoSchema())) @@ -70,11 +70,10 @@ func TestDisableAutoAnalyze(t *testing.T) { is := dom.InfoSchema() require.NoError(t, h.Update(is)) - // Set auto analyze ratio to 0. - tk.MustExec("set @@global.tidb_auto_analyze_ratio = 0") - autoanalyze.AutoAnalyzeMinCnt = 0 + tk.MustExec("set @@global.tidb_enable_auto_analyze = 0") + statistics.AutoAnalyzeMinCnt = 0 defer func() { - autoanalyze.AutoAnalyzeMinCnt = 1000 + statistics.AutoAnalyzeMinCnt = 1000 }() // Even auto analyze ratio is set to 0, we still need to analyze the unanalyzed tables. require.True(t, dom.StatsHandle().HandleAutoAnalyze(dom.InfoSchema())) @@ -97,9 +96,9 @@ func TestAutoAnalyzeOnChangeAnalyzeVer(t *testing.T) { tk.MustExec("insert into t values(1)") tk.MustExec("set @@global.tidb_analyze_version = 1") do := dom - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 defer func() { - autoanalyze.AutoAnalyzeMinCnt = 1000 + statistics.AutoAnalyzeMinCnt = 1000 }() h := do.StatsHandle() err := h.HandleDDLEvent(<-h.DDLEventCh()) @@ -286,10 +285,10 @@ func TestAutoAnalyzeSkipColumnTypes(t *testing.T) { require.NoError(t, h.Update(dom.InfoSchema())) tk.MustExec("set @@global.tidb_analyze_skip_column_types = 'json,blob,mediumblob,text,mediumtext'") - originalVal := autoanalyze.AutoAnalyzeMinCnt - autoanalyze.AutoAnalyzeMinCnt = 0 + originalVal := statistics.AutoAnalyzeMinCnt + statistics.AutoAnalyzeMinCnt = 0 defer func() { - autoanalyze.AutoAnalyzeMinCnt = originalVal + statistics.AutoAnalyzeMinCnt = originalVal }() require.True(t, h.HandleAutoAnalyze(dom.InfoSchema())) tk.MustQuery("select job_info from mysql.analyze_jobs where job_info like '%auto analyze table%'").Check(testkit.Rows("auto analyze table columns a, b, d with 256 buckets, 500 topn, 1 samplerate")) @@ -318,7 +317,7 @@ func TestAutoAnalyzeOnEmptyTable(t *testing.T) { // to pass the stats.Pseudo check in autoAnalyzeTable tk.MustExec("analyze table t") // to pass the AutoAnalyzeMinCnt check in autoAnalyzeTable - tk.MustExec("insert into t values (1)" + strings.Repeat(", (1)", int(autoanalyze.AutoAnalyzeMinCnt))) + tk.MustExec("insert into t values (1)" + strings.Repeat(", (1)", int(statistics.AutoAnalyzeMinCnt))) require.NoError(t, dom.StatsHandle().DumpStatsDeltaToKV(true)) require.NoError(t, dom.StatsHandle().Update(dom.InfoSchema())) @@ -353,7 +352,7 @@ func TestAutoAnalyzeOutOfSpecifiedTime(t *testing.T) { // to pass the stats.Pseudo check in autoAnalyzeTable tk.MustExec("analyze table t") // to pass the AutoAnalyzeMinCnt check in autoAnalyzeTable - tk.MustExec("insert into t values (1)" + strings.Repeat(", (1)", int(autoanalyze.AutoAnalyzeMinCnt))) + tk.MustExec("insert into t values (1)" + strings.Repeat(", (1)", int(statistics.AutoAnalyzeMinCnt))) require.NoError(t, dom.StatsHandle().DumpStatsDeltaToKV(true)) require.NoError(t, dom.StatsHandle().Update(dom.InfoSchema())) diff --git a/pkg/statistics/handle/cache/statscache.go b/pkg/statistics/handle/cache/statscache.go index 3f3e8d71017a6..8cb26d710af5f 100644 --- a/pkg/statistics/handle/cache/statscache.go +++ b/pkg/statistics/handle/cache/statscache.go @@ -200,7 +200,14 @@ func (s *StatsCacheImpl) SetStatsCacheCapacity(c int64) { // UpdateStatsHealthyMetrics updates stats healthy distribution metrics according to stats cache. func (s *StatsCacheImpl) UpdateStatsHealthyMetrics() { distribution := make([]int64, 5) + uneligibleAnalyze := 0 for _, tbl := range s.Values() { + distribution[4]++ // total table count + isEligibleForAnalysis := tbl.IsEligibleForAnalysis() + if !isEligibleForAnalysis { + uneligibleAnalyze++ + continue + } healthy, ok := tbl.GetStatsHealthy() if !ok { continue @@ -214,9 +221,9 @@ func (s *StatsCacheImpl) UpdateStatsHealthyMetrics() { } else { distribution[3]++ } - distribution[4]++ } for i, val := range distribution { handle_metrics.StatsHealthyGauges[i].Set(float64(val)) } + handle_metrics.StatsHealthyGauges[5].Set(float64(uneligibleAnalyze)) } diff --git a/pkg/statistics/handle/metrics/metrics.go b/pkg/statistics/handle/metrics/metrics.go index 175fef6359df4..c4fd3a644546d 100644 --- a/pkg/statistics/handle/metrics/metrics.go +++ b/pkg/statistics/handle/metrics/metrics.go @@ -40,6 +40,7 @@ func InitMetricsVars() { metrics.StatsHealthyGauge.WithLabelValues("[100,100]"), // [0,100] should always be the last metrics.StatsHealthyGauge.WithLabelValues("[0,100]"), + metrics.StatsHealthyGauge.WithLabelValues("unneeded analyze"), } DumpHistoricalStatsSuccessCounter = metrics.HistoricalStatsCounter.WithLabelValues("dump", "success") diff --git a/pkg/statistics/handle/updatetest/BUILD.bazel b/pkg/statistics/handle/updatetest/BUILD.bazel index 6b08edca6b991..edfeeb5a0d9a1 100644 --- a/pkg/statistics/handle/updatetest/BUILD.bazel +++ b/pkg/statistics/handle/updatetest/BUILD.bazel @@ -16,7 +16,6 @@ go_test( "//pkg/sessionctx", "//pkg/sessionctx/variable", "//pkg/statistics", - "//pkg/statistics/handle/autoanalyze", "//pkg/statistics/handle/usage", "//pkg/statistics/handle/util", "//pkg/testkit", diff --git a/pkg/statistics/handle/updatetest/update_test.go b/pkg/statistics/handle/updatetest/update_test.go index 01ef3342f6aea..88bade3579df5 100644 --- a/pkg/statistics/handle/updatetest/update_test.go +++ b/pkg/statistics/handle/updatetest/update_test.go @@ -28,7 +28,6 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/statistics" - "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze" "github.com/pingcap/tidb/pkg/statistics/handle/usage" "github.com/pingcap/tidb/pkg/statistics/handle/util" "github.com/pingcap/tidb/pkg/testkit" @@ -363,11 +362,11 @@ func TestAutoUpdate(t *testing.T) { testKit.MustExec("use test") testKit.MustExec("create table t (a varchar(20))") - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 testKit.MustExec("set global tidb_auto_analyze_ratio = 0.2") defer func() { - autoanalyze.AutoAnalyzeMinCnt = 1000 - testKit.MustExec("set global tidb_auto_analyze_ratio = 0.0") + statistics.AutoAnalyzeMinCnt = 1000 + testKit.MustExec("set global tidb_auto_analyze_ratio = 0.5") }() do := dom @@ -468,11 +467,11 @@ func TestAutoUpdatePartition(t *testing.T) { testKit.MustExec("create table t (a int) PARTITION BY RANGE (a) (PARTITION p0 VALUES LESS THAN (6))") testKit.MustExec("analyze table t") - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 testKit.MustExec("set global tidb_auto_analyze_ratio = 0.6") defer func() { - autoanalyze.AutoAnalyzeMinCnt = 1000 - testKit.MustExec("set global tidb_auto_analyze_ratio = 0.0") + statistics.AutoAnalyzeMinCnt = 1000 + testKit.MustExec("set global tidb_auto_analyze_ratio = 0.5") }() do := dom @@ -513,7 +512,7 @@ func TestIssue25700(t *testing.T) { tk.MustExec("drop table if exists t") tk.MustExec("CREATE TABLE `t` ( `ldecimal` decimal(32,4) DEFAULT NULL, `rdecimal` decimal(32,4) DEFAULT NULL, `gen_col` decimal(36,4) GENERATED ALWAYS AS (`ldecimal` + `rdecimal`) VIRTUAL, `col_timestamp` timestamp(3) NULL DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;") tk.MustExec("analyze table t") - tk.MustExec("INSERT INTO `t` (`ldecimal`, `rdecimal`, `col_timestamp`) VALUES (2265.2200, 9843.4100, '1999-12-31 16:00:00')" + strings.Repeat(", (2265.2200, 9843.4100, '1999-12-31 16:00:00')", int(autoanalyze.AutoAnalyzeMinCnt))) + tk.MustExec("INSERT INTO `t` (`ldecimal`, `rdecimal`, `col_timestamp`) VALUES (2265.2200, 9843.4100, '1999-12-31 16:00:00')" + strings.Repeat(", (2265.2200, 9843.4100, '1999-12-31 16:00:00')", int(statistics.AutoAnalyzeMinCnt))) require.NoError(t, dom.StatsHandle().DumpStatsDeltaToKV(true)) require.NoError(t, dom.StatsHandle().Update(dom.InfoSchema())) @@ -803,11 +802,11 @@ func TestAutoUpdatePartitionInDynamicOnlyMode(t *testing.T) { testKit.MustExec("set @@tidb_analyze_version = 2") testKit.MustExec("analyze table t") - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 testKit.MustExec("set global tidb_auto_analyze_ratio = 0.1") defer func() { - autoanalyze.AutoAnalyzeMinCnt = 1000 - testKit.MustExec("set global tidb_auto_analyze_ratio = 0.0") + statistics.AutoAnalyzeMinCnt = 1000 + testKit.MustExec("set global tidb_auto_analyze_ratio = 0.5") }() require.NoError(t, h.Update(is)) @@ -849,9 +848,9 @@ func TestAutoAnalyzeRatio(t *testing.T) { oriStart := tk.MustQuery("select @@tidb_auto_analyze_start_time").Rows()[0][0].(string) oriEnd := tk.MustQuery("select @@tidb_auto_analyze_end_time").Rows()[0][0].(string) - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 defer func() { - autoanalyze.AutoAnalyzeMinCnt = 1000 + statistics.AutoAnalyzeMinCnt = 1000 tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_start_time='%v'", oriStart)) tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_end_time='%v'", oriEnd)) }() @@ -1077,9 +1076,9 @@ func TestStatsLockUnlockForAutoAnalyze(t *testing.T) { oriStart := tk.MustQuery("select @@tidb_auto_analyze_start_time").Rows()[0][0].(string) oriEnd := tk.MustQuery("select @@tidb_auto_analyze_end_time").Rows()[0][0].(string) - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 defer func() { - autoanalyze.AutoAnalyzeMinCnt = 1000 + statistics.AutoAnalyzeMinCnt = 1000 tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_start_time='%v'", oriStart)) tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_end_time='%v'", oriEnd)) }() @@ -1277,15 +1276,15 @@ func TestNotDumpSysTable(t *testing.T) { func TestAutoAnalyzePartitionTableAfterAddingIndex(t *testing.T) { store, dom := testkit.CreateMockStoreAndDomain(t) tk := testkit.NewTestKit(t, store) - oriMinCnt := autoanalyze.AutoAnalyzeMinCnt + oriMinCnt := statistics.AutoAnalyzeMinCnt oriStart := tk.MustQuery("select @@tidb_auto_analyze_start_time").Rows()[0][0].(string) oriEnd := tk.MustQuery("select @@tidb_auto_analyze_end_time").Rows()[0][0].(string) defer func() { - autoanalyze.AutoAnalyzeMinCnt = oriMinCnt + statistics.AutoAnalyzeMinCnt = oriMinCnt tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_start_time='%v'", oriStart)) tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_end_time='%v'", oriEnd)) }() - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 tk.MustExec("set global tidb_auto_analyze_start_time='00:00 +0000'") tk.MustExec("set global tidb_auto_analyze_end_time='23:59 +0000'") tk.MustExec("set global tidb_analyze_version = 2") diff --git a/pkg/statistics/integration_test.go b/pkg/statistics/integration_test.go index c675cd47f857c..7a01b53b52a05 100644 --- a/pkg/statistics/integration_test.go +++ b/pkg/statistics/integration_test.go @@ -26,7 +26,6 @@ import ( "github.com/pingcap/tidb/pkg/config" "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/statistics" - "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze" "github.com/pingcap/tidb/pkg/testkit" "github.com/pingcap/tidb/pkg/testkit/testdata" "github.com/stretchr/testify/require" @@ -307,9 +306,9 @@ func TestOutdatedStatsCheck(t *testing.T) { oriStart := tk.MustQuery("select @@tidb_auto_analyze_start_time").Rows()[0][0].(string) oriEnd := tk.MustQuery("select @@tidb_auto_analyze_end_time").Rows()[0][0].(string) - autoanalyze.AutoAnalyzeMinCnt = 0 + statistics.AutoAnalyzeMinCnt = 0 defer func() { - autoanalyze.AutoAnalyzeMinCnt = 1000 + statistics.AutoAnalyzeMinCnt = 1000 tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_start_time='%v'", oriStart)) tk.MustExec(fmt.Sprintf("set global tidb_auto_analyze_end_time='%v'", oriEnd)) }() diff --git a/pkg/statistics/table.go b/pkg/statistics/table.go index ac8645adbe87b..3112b4516f341 100644 --- a/pkg/statistics/table.go +++ b/pkg/statistics/table.go @@ -42,6 +42,10 @@ const ( PseudoRowCount = 10000 ) +// AutoAnalyzeMinCnt means if the count of table is less than this value, we don't need to do auto analyze. +// Exported for testing. +var AutoAnalyzeMinCnt int64 = 1000 + var ( // Below functions are used to solve cycle import problem. // Note: all functions below will be removed after finishing moving all estimation functions into the cardinality package. @@ -535,6 +539,19 @@ func (t *Table) ReleaseAndPutToPool() { maps.Clear(t.Indices) } +// IsEligibleForAnalysis checks whether the table is eligible for analysis. +func (t *Table) IsEligibleForAnalysis() bool { + // 1. If the statistics are either not loaded or are classified as pseudo, there is no need for analyze. + // Pseudo statistics can be created by the optimizer, so we need to double check it. + // 2. If the table is too small, we don't want to waste time to analyze it. + // Leave the opportunity to other bigger tables. + if t == nil || t.Pseudo || t.RealtimeCount < AutoAnalyzeMinCnt { + return false + } + + return true +} + // ID2UniqueID generates a new HistColl whose `Columns` is built from UniqueID of given columns. func (coll *HistColl) ID2UniqueID(columns []*expression.Column) *HistColl { cols := make(map[int64]*Column) diff --git a/pkg/ttl/ttlworker/BUILD.bazel b/pkg/ttl/ttlworker/BUILD.bazel index 6162c2d25ab53..469818125658d 100644 --- a/pkg/ttl/ttlworker/BUILD.bazel +++ b/pkg/ttl/ttlworker/BUILD.bazel @@ -81,7 +81,7 @@ go_test( "//pkg/session", "//pkg/sessionctx", "//pkg/sessionctx/variable", - "//pkg/statistics/handle/autoanalyze", + "//pkg/statistics", "//pkg/store/mockstore", "//pkg/testkit", "//pkg/timer/api", diff --git a/pkg/ttl/ttlworker/job_manager_integration_test.go b/pkg/ttl/ttlworker/job_manager_integration_test.go index 00ec19e8dfddd..a8b9a81ba10ff 100644 --- a/pkg/ttl/ttlworker/job_manager_integration_test.go +++ b/pkg/ttl/ttlworker/job_manager_integration_test.go @@ -32,7 +32,7 @@ import ( "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/model" dbsession "github.com/pingcap/tidb/pkg/session" - "github.com/pingcap/tidb/pkg/statistics/handle/autoanalyze" + "github.com/pingcap/tidb/pkg/statistics" "github.com/pingcap/tidb/pkg/testkit" timerapi "github.com/pingcap/tidb/pkg/timer/api" timertable "github.com/pingcap/tidb/pkg/timer/tablestore" @@ -178,10 +178,10 @@ func TestTTLAutoAnalyze(t *testing.T) { failpoint.Enable("github.com/pingcap/tidb/pkg/ttl/ttlworker/task-manager-loop-interval", fmt.Sprintf("return(%d)", time.Second)) defer failpoint.Disable("github.com/pingcap/tidb/pkg/ttl/ttlworker/task-manager-loop-interval") - originAutoAnalyzeMinCnt := autoanalyze.AutoAnalyzeMinCnt - autoanalyze.AutoAnalyzeMinCnt = 0 + originAutoAnalyzeMinCnt := statistics.AutoAnalyzeMinCnt + statistics.AutoAnalyzeMinCnt = 0 defer func() { - autoanalyze.AutoAnalyzeMinCnt = originAutoAnalyzeMinCnt + statistics.AutoAnalyzeMinCnt = originAutoAnalyzeMinCnt }() store, dom := testkit.CreateMockStoreAndDomain(t) @@ -364,10 +364,10 @@ func TestTTLJobDisable(t *testing.T) { failpoint.Enable("github.com/pingcap/tidb/pkg/ttl/ttlworker/resize-workers-interval", fmt.Sprintf("return(%d)", time.Second)) defer failpoint.Disable("github.com/pingcap/tidb/pkg/ttl/ttlworker/resize-workers-interval") - originAutoAnalyzeMinCnt := autoanalyze.AutoAnalyzeMinCnt - autoanalyze.AutoAnalyzeMinCnt = 0 + originAutoAnalyzeMinCnt := statistics.AutoAnalyzeMinCnt + statistics.AutoAnalyzeMinCnt = 0 defer func() { - autoanalyze.AutoAnalyzeMinCnt = originAutoAnalyzeMinCnt + statistics.AutoAnalyzeMinCnt = originAutoAnalyzeMinCnt }() store, dom := testkit.CreateMockStoreAndDomain(t)