diff --git a/DEPS.bzl b/DEPS.bzl index 4aa142c8d2c32..1071551837003 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -6894,26 +6894,26 @@ def go_deps(): name = "com_github_tikv_client_go_v2", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/client-go/v2", - sha256 = "b0a067425a329c47e5d34d40eb3390a8b141ff1d3d4b19ee3a9d05aa4168c165", - strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20251202113147-065a1251aea9", + sha256 = "2ec5b60a15a7d41ac18cc8d481cba5f7b42cbf9a883df6340907b413a1d28128", + strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20251211093144-53f3fad17830", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251202113147-065a1251aea9.zip", - "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251202113147-065a1251aea9.zip", - "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251202113147-065a1251aea9.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251202113147-065a1251aea9.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251211093144-53f3fad17830.zip", + "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251211093144-53f3fad17830.zip", + "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251211093144-53f3fad17830.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251211093144-53f3fad17830.zip", ], ) go_repository( name = "com_github_tikv_pd_client", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/pd/client", - sha256 = "c06a8fc846b03bcbddf902560aa86235f746bb603a4bbd624399d770d505e858", - strip_prefix = "github.com/tikv/pd/client@v0.0.0-20250901062501-1646b924d286", + sha256 = "eccb8dab31077423e7dc7bacf95f350c9584bb166b5641b98322c88bfd85d891", + strip_prefix = "github.com/tikv/pd/client@v0.0.0-20251211090623-d08fd48c229d", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250901062501-1646b924d286.zip", - "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250901062501-1646b924d286.zip", - "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250901062501-1646b924d286.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250901062501-1646b924d286.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20251211090623-d08fd48c229d.zip", + "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20251211090623-d08fd48c229d.zip", + "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20251211090623-d08fd48c229d.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20251211090623-d08fd48c229d.zip", ], ) go_repository( diff --git a/go.mod b/go.mod index fdf75a44da480..bc89c9a5b9882 100644 --- a/go.mod +++ b/go.mod @@ -110,8 +110,8 @@ require ( github.com/stretchr/testify v1.10.0 github.com/tdakkota/asciicheck v0.2.0 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 - github.com/tikv/client-go/v2 v2.0.8-0.20251202113147-065a1251aea9 - github.com/tikv/pd/client v0.0.0-20250901062501-1646b924d286 + github.com/tikv/client-go/v2 v2.0.8-0.20251211093144-53f3fad17830 + github.com/tikv/pd/client v0.0.0-20251211090623-d08fd48c229d github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a github.com/twmb/murmur3 v1.1.6 github.com/uber/jaeger-client-go v2.22.1+incompatible diff --git a/go.sum b/go.sum index c319073de78f7..3e64a5756d0ff 100644 --- a/go.sum +++ b/go.sum @@ -760,10 +760,10 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJf github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= -github.com/tikv/client-go/v2 v2.0.8-0.20251202113147-065a1251aea9 h1:QRJCyuB3vb/RHLf/9KDXRySAK1zB8rFInfmcHAchl3k= -github.com/tikv/client-go/v2 v2.0.8-0.20251202113147-065a1251aea9/go.mod h1:DrneTzlX66kdfl59PMdaHpeJWzyXXNClNqzhi71AvXc= -github.com/tikv/pd/client v0.0.0-20250901062501-1646b924d286 h1:TBrJ7eyjLfI6xc7rr348sTKPUa96ZUWmfSonqLP0vVM= -github.com/tikv/pd/client v0.0.0-20250901062501-1646b924d286/go.mod h1:kuIEDRLck7LGHiqKYrQR3fNiK06trLmmK02s4r99iWU= +github.com/tikv/client-go/v2 v2.0.8-0.20251211093144-53f3fad17830 h1:/A9JXuDKg6WuCTbCrfJs9KxyCM8wXmo2asGTg5cLaHw= +github.com/tikv/client-go/v2 v2.0.8-0.20251211093144-53f3fad17830/go.mod h1:gipu6QUanLH6VCf15eTVrGpkI3fqmdqmlRIyk4PSKyU= +github.com/tikv/pd/client v0.0.0-20251211090623-d08fd48c229d h1:bQGUYqYNn8vW9/3Oh/3RvpqX5HKXO+9ii32cfy/Uh8o= +github.com/tikv/pd/client v0.0.0-20251211090623-d08fd48c229d/go.mod h1:WnwkeHrbTE0a44TtmTY6jZZ1dx8eU4O5uSqLeXjXrOc= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a h1:A6uKudFIfAEpoPdaal3aSqGxBzLyU8TqyXImLwo6dIo= github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= diff --git a/pkg/domain/BUILD.bazel b/pkg/domain/BUILD.bazel index be24d3d7bd2eb..f9fb2a9231bd5 100644 --- a/pkg/domain/BUILD.bazel +++ b/pkg/domain/BUILD.bazel @@ -109,6 +109,7 @@ go_library( "@com_github_tikv_client_go_v2//tikv", "@com_github_tikv_client_go_v2//txnkv/transaction", "@com_github_tikv_pd_client//:client", + "@com_github_tikv_pd_client//circuitbreaker", "@com_github_tikv_pd_client//http", "@com_github_tikv_pd_client//resource_group/controller", "@io_etcd_go_etcd_client_v3//:client", diff --git a/pkg/domain/domain_sysvars.go b/pkg/domain/domain_sysvars.go index 0c540c323ca44..6b00131d9dd3e 100644 --- a/pkg/domain/domain_sysvars.go +++ b/pkg/domain/domain_sysvars.go @@ -22,7 +22,9 @@ import ( "github.com/pingcap/tidb/pkg/kv" "github.com/pingcap/tidb/pkg/meta" "github.com/pingcap/tidb/pkg/sessionctx/variable" + "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" + "github.com/tikv/pd/client/circuitbreaker" ) // initDomainSysVars() is called when a domain is initialized. @@ -44,6 +46,8 @@ func (do *Domain) initDomainSysVars() { variable.SetLowResolutionTSOUpdateInterval = do.setLowResolutionTSOUpdateInterval variable.ChangeSchemaCacheSize = do.changeSchemaCacheSize + + variable.ChangePDMetadataCircuitBreakerErrorRateThresholdRatio = changePDMetadataCircuitBreakerErrorRateThresholdRatio } // setStatsCacheCapacity sets statsCache cap @@ -149,3 +153,9 @@ func (do *Domain) changeSchemaCacheSize(ctx context.Context, size uint64) error do.infoCache.Data.SetCacheCapacity(size) return nil } + +func changePDMetadataCircuitBreakerErrorRateThresholdRatio(errorRateRatio uint32) { + tikv.ChangePDRegionMetaCircuitBreakerSettings(func(config *circuitbreaker.Settings) { + config.ErrorRateThresholdPct = errorRateRatio + }) +} diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index 6378e5417f1d0..947ddeff685b2 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -3551,6 +3551,23 @@ var defaultSysVars = []*SysVar{ return AdvancerCheckPointLagLimit.Load().String(), nil }, }, + {Scope: ScopeGlobal, Name: TiDBCircuitBreakerPDMetadataErrorRateThresholdRatio, Value: strconv.FormatFloat(DefTiDBCircuitBreakerPDMetaErrorRateRatio, 'f', -1, 64), + Type: TypeFloat, MinValue: 0, MaxValue: 1, + GetGlobal: func(_ context.Context, s *SessionVars) (string, error) { + return strconv.FormatFloat(CircuitBreakerPDMetadataErrorRateThresholdRatio.Load(), 'f', -1, 64), nil + }, + SetGlobal: func(_ context.Context, s *SessionVars, val string) error { + v := tidbOptFloat64(val, DefTiDBCircuitBreakerPDMetaErrorRateRatio) + if v < 0 || v > 1 { + return errors.Errorf("invalid tidb_cb_pd_metadata_error_rate_threshold_ratio value %s", val) + } + CircuitBreakerPDMetadataErrorRateThresholdRatio.Store(v) + if ChangePDMetadataCircuitBreakerErrorRateThresholdRatio != nil { + ChangePDMetadataCircuitBreakerErrorRateThresholdRatio(uint32(v * 100)) + } + return nil + }, + }, } // GlobalSystemVariableInitialValue gets the default value for a system variable including ones that are dynamically set (e.g. based on the store) diff --git a/pkg/sessionctx/variable/sysvar_test.go b/pkg/sessionctx/variable/sysvar_test.go index 5dd0b43367f6c..815bd6b4e1c47 100644 --- a/pkg/sessionctx/variable/sysvar_test.go +++ b/pkg/sessionctx/variable/sysvar_test.go @@ -1742,6 +1742,34 @@ func TestTiDBSchemaCacheSize(t *testing.T) { require.Error(t, err) } +func TestTiDBCircuitBreakerPDMetadataErrorRateThresholdRatio(t *testing.T) { + sv := GetSysVar(TiDBCircuitBreakerPDMetadataErrorRateThresholdRatio) + vars := NewSessionVars(nil) + + // Too low, will get raised to the min value + val, err := sv.Validate(vars, "-1", ScopeGlobal) + require.NoError(t, err) + require.Equal(t, strconv.FormatInt(GetSysVar(TiDBCircuitBreakerPDMetadataErrorRateThresholdRatio).MinValue, 10), val) + warn := vars.StmtCtx.GetWarnings()[0].Err + require.Equal(t, "[variable:1292]Truncated incorrect tidb_cb_pd_metadata_error_rate_threshold_ratio value: '-1'", warn.Error()) + + // Too high, will get lowered to the max value + val, err = sv.Validate(vars, "1.1", ScopeGlobal) + require.NoError(t, err) + require.Equal(t, strconv.FormatUint(GetSysVar(TiDBCircuitBreakerPDMetadataErrorRateThresholdRatio).MaxValue, 10), val) + warn = vars.StmtCtx.GetWarnings()[1].Err + require.Equal(t, "[variable:1292]Truncated incorrect tidb_cb_pd_metadata_error_rate_threshold_ratio value: '1.1'", warn.Error()) + + // valid + val, err = sv.Validate(vars, "0.9", ScopeGlobal) + require.NoError(t, err) + require.Equal(t, "0.9", val) + + val, err = sv.Validate(vars, "0.0", ScopeGlobal) + require.NoError(t, err) + require.Equal(t, "0.0", val) +} + func TestEnableWindowFunction(t *testing.T) { vars := NewSessionVars(nil) require.Equal(t, vars.EnableWindowFunction, DefEnableWindowFunction) diff --git a/pkg/sessionctx/variable/tidb_vars.go b/pkg/sessionctx/variable/tidb_vars.go index a699de2da7f89..5aef53c961ec7 100644 --- a/pkg/sessionctx/variable/tidb_vars.go +++ b/pkg/sessionctx/variable/tidb_vars.go @@ -1250,6 +1250,9 @@ const ( // TiDBAdvancerCheckPointLagLimit controls the maximum lag could be tolerated for the checkpoint lag. // The log backup task will be paused if the checkpoint lag is larger than it. TiDBAdvancerCheckPointLagLimit = "tidb_advancer_check_point_lag_limit" + // TiDBCircuitBreakerPDMetadataErrorRateThresholdRatio variable is used to set ratio of errors to trip the circuit breaker for get region calls to PD + // https://github.com/tikv/rfcs/blob/master/text/0115-circuit-breaker.md + TiDBCircuitBreakerPDMetadataErrorRateThresholdRatio = "tidb_cb_pd_metadata_error_rate_threshold_ratio" ) // TiDB intentional limits @@ -1622,6 +1625,7 @@ const ( DefTiDBTSOClientRPCMode = TSOClientRPCModeDefault DefTiDBLoadBindingTimeout = 200 DefTiDBAdvancerCheckPointLagLimit = 48 * time.Hour + DefTiDBCircuitBreakerPDMetaErrorRateRatio = 0.0 ) // Process global variables. @@ -1745,7 +1749,8 @@ var ( SchemaCacheSize = atomic.NewUint64(DefTiDBSchemaCacheSize) SchemaCacheSizeOriginText = atomic.NewString(strconv.Itoa(DefTiDBSchemaCacheSize)) - AdvancerCheckPointLagLimit = atomic.NewDuration(DefTiDBAdvancerCheckPointLagLimit) + AdvancerCheckPointLagLimit = atomic.NewDuration(DefTiDBAdvancerCheckPointLagLimit) + CircuitBreakerPDMetadataErrorRateThresholdRatio = atomic.NewFloat64(0.0) ) var ( @@ -1781,6 +1786,8 @@ var ( EnableStatsOwner func() error = nil // DisableStatsOwner is the func registered by stats to disable running stats in this instance. DisableStatsOwner func() error = nil + // ChangePDMetadataCircuitBreakerErrorRateThresholdRatio changes the error rate threshold of the PD metadata circuit breaker. + ChangePDMetadataCircuitBreakerErrorRateThresholdRatio func(uint32) = nil ) // Hooks functions for Cluster Resource Control.