Skip to content

*: integrate circuitbreaker for get region calls to PD (#58737) #64970

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ti-chi-bot merged 6 commits into from
Dec 16, 2025
Merged

*: integrate circuitbreaker for get region calls to PD (#58737) #64970

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b2eedbc
This is an automated cherry-pick of #58737
Tema Jan 24, 2025
a302198
resolve conflicts
rleungx Dec 11, 2025
409aee2
fix
rleungx Dec 11, 2025
0da01da
session: rename circuit breaker sysvar (#61951)
rleungx Jun 24, 2025
9c023bc
session: change the variable range of circuit breaker (#61613)
rleungx Jun 10, 2025
6a0b637
fix
rleungx Dec 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions DEPS.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6894,26 +6894,26 @@ def go_deps():
name = "com_github_tikv_client_go_v2",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/client-go/v2",
sha256 = "b0a067425a329c47e5d34d40eb3390a8b141ff1d3d4b19ee3a9d05aa4168c165",
strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20251202113147-065a1251aea9",
sha256 = "2ec5b60a15a7d41ac18cc8d481cba5f7b42cbf9a883df6340907b413a1d28128",
strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20251211093144-53f3fad17830",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251202113147-065a1251aea9.zip",
"http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251202113147-065a1251aea9.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251202113147-065a1251aea9.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251202113147-065a1251aea9.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251211093144-53f3fad17830.zip",
"http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251211093144-53f3fad17830.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251211093144-53f3fad17830.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20251211093144-53f3fad17830.zip",
],
)
go_repository(
name = "com_github_tikv_pd_client",
build_file_proto_mode = "disable_global",
importpath = "github.com/tikv/pd/client",
sha256 = "c06a8fc846b03bcbddf902560aa86235f746bb603a4bbd624399d770d505e858",
strip_prefix = "github.com/tikv/pd/client@v0.0.0-20250901062501-1646b924d286",
sha256 = "eccb8dab31077423e7dc7bacf95f350c9584bb166b5641b98322c88bfd85d891",
strip_prefix = "github.com/tikv/pd/client@v0.0.0-20251211090623-d08fd48c229d",
urls = [
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250901062501-1646b924d286.zip",
"http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250901062501-1646b924d286.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250901062501-1646b924d286.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20250901062501-1646b924d286.zip",
"http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20251211090623-d08fd48c229d.zip",
"http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20251211090623-d08fd48c229d.zip",
"https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20251211090623-d08fd48c229d.zip",
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20251211090623-d08fd48c229d.zip",
],
)
go_repository(
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ require (
github.com/stretchr/testify v1.10.0
github.com/tdakkota/asciicheck v0.2.0
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
github.com/tikv/client-go/v2 v2.0.8-0.20251202113147-065a1251aea9
github.com/tikv/pd/client v0.0.0-20250901062501-1646b924d286
github.com/tikv/client-go/v2 v2.0.8-0.20251211093144-53f3fad17830
github.com/tikv/pd/client v0.0.0-20251211090623-d08fd48c229d
github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a
github.com/twmb/murmur3 v1.1.6
github.com/uber/jaeger-client-go v2.22.1+incompatible
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -760,10 +760,10 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJf
github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU=
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
github.com/tikv/client-go/v2 v2.0.8-0.20251202113147-065a1251aea9 h1:QRJCyuB3vb/RHLf/9KDXRySAK1zB8rFInfmcHAchl3k=
github.com/tikv/client-go/v2 v2.0.8-0.20251202113147-065a1251aea9/go.mod h1:DrneTzlX66kdfl59PMdaHpeJWzyXXNClNqzhi71AvXc=
github.com/tikv/pd/client v0.0.0-20250901062501-1646b924d286 h1:TBrJ7eyjLfI6xc7rr348sTKPUa96ZUWmfSonqLP0vVM=
github.com/tikv/pd/client v0.0.0-20250901062501-1646b924d286/go.mod h1:kuIEDRLck7LGHiqKYrQR3fNiK06trLmmK02s4r99iWU=
github.com/tikv/client-go/v2 v2.0.8-0.20251211093144-53f3fad17830 h1:/A9JXuDKg6WuCTbCrfJs9KxyCM8wXmo2asGTg5cLaHw=
github.com/tikv/client-go/v2 v2.0.8-0.20251211093144-53f3fad17830/go.mod h1:gipu6QUanLH6VCf15eTVrGpkI3fqmdqmlRIyk4PSKyU=
github.com/tikv/pd/client v0.0.0-20251211090623-d08fd48c229d h1:bQGUYqYNn8vW9/3Oh/3RvpqX5HKXO+9ii32cfy/Uh8o=
github.com/tikv/pd/client v0.0.0-20251211090623-d08fd48c229d/go.mod h1:WnwkeHrbTE0a44TtmTY6jZZ1dx8eU4O5uSqLeXjXrOc=
github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a h1:A6uKudFIfAEpoPdaal3aSqGxBzLyU8TqyXImLwo6dIo=
github.com/timakin/bodyclose v0.0.0-20240125160201-f835fa56326a/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460=
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
Expand Down
1 change: 1 addition & 0 deletions pkg/domain/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ go_library(
"@com_github_tikv_client_go_v2//tikv",
"@com_github_tikv_client_go_v2//txnkv/transaction",
"@com_github_tikv_pd_client//:client",
"@com_github_tikv_pd_client//circuitbreaker",
"@com_github_tikv_pd_client//http",
"@com_github_tikv_pd_client//resource_group/controller",
"@io_etcd_go_etcd_client_v3//:client",
Expand Down
10 changes: 10 additions & 0 deletions pkg/domain/domain_sysvars.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ import (
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/meta"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/tikv/client-go/v2/tikv"
pd "github.com/tikv/pd/client"
"github.com/tikv/pd/client/circuitbreaker"
)

// initDomainSysVars() is called when a domain is initialized.
Expand All @@ -44,6 +46,8 @@ func (do *Domain) initDomainSysVars() {
variable.SetLowResolutionTSOUpdateInterval = do.setLowResolutionTSOUpdateInterval

variable.ChangeSchemaCacheSize = do.changeSchemaCacheSize

variable.ChangePDMetadataCircuitBreakerErrorRateThresholdRatio = changePDMetadataCircuitBreakerErrorRateThresholdRatio
}

// setStatsCacheCapacity sets statsCache cap
Expand Down Expand Up @@ -149,3 +153,9 @@ func (do *Domain) changeSchemaCacheSize(ctx context.Context, size uint64) error
do.infoCache.Data.SetCacheCapacity(size)
return nil
}

func changePDMetadataCircuitBreakerErrorRateThresholdRatio(errorRateRatio uint32) {
tikv.ChangePDRegionMetaCircuitBreakerSettings(func(config *circuitbreaker.Settings) {
config.ErrorRateThresholdPct = errorRateRatio
})
}
17 changes: 17 additions & 0 deletions pkg/sessionctx/variable/sysvar.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3551,6 +3551,23 @@ var defaultSysVars = []*SysVar{
return AdvancerCheckPointLagLimit.Load().String(), nil
},
},
{Scope: ScopeGlobal, Name: TiDBCircuitBreakerPDMetadataErrorRateThresholdRatio, Value: strconv.FormatFloat(DefTiDBCircuitBreakerPDMetaErrorRateRatio, 'f', -1, 64),
Type: TypeFloat, MinValue: 0, MaxValue: 1,
GetGlobal: func(_ context.Context, s *SessionVars) (string, error) {
return strconv.FormatFloat(CircuitBreakerPDMetadataErrorRateThresholdRatio.Load(), 'f', -1, 64), nil
},
SetGlobal: func(_ context.Context, s *SessionVars, val string) error {
v := tidbOptFloat64(val, DefTiDBCircuitBreakerPDMetaErrorRateRatio)
if v < 0 || v > 1 {
return errors.Errorf("invalid tidb_cb_pd_metadata_error_rate_threshold_ratio value %s", val)
}
CircuitBreakerPDMetadataErrorRateThresholdRatio.Store(v)
if ChangePDMetadataCircuitBreakerErrorRateThresholdRatio != nil {
ChangePDMetadataCircuitBreakerErrorRateThresholdRatio(uint32(v * 100))
}
return nil
},
},
}

// GlobalSystemVariableInitialValue gets the default value for a system variable including ones that are dynamically set (e.g. based on the store)
Expand Down
28 changes: 28 additions & 0 deletions pkg/sessionctx/variable/sysvar_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1742,6 +1742,34 @@ func TestTiDBSchemaCacheSize(t *testing.T) {
require.Error(t, err)
}

func TestTiDBCircuitBreakerPDMetadataErrorRateThresholdRatio(t *testing.T) {
sv := GetSysVar(TiDBCircuitBreakerPDMetadataErrorRateThresholdRatio)
vars := NewSessionVars(nil)

// Too low, will get raised to the min value
val, err := sv.Validate(vars, "-1", ScopeGlobal)
require.NoError(t, err)
require.Equal(t, strconv.FormatInt(GetSysVar(TiDBCircuitBreakerPDMetadataErrorRateThresholdRatio).MinValue, 10), val)
warn := vars.StmtCtx.GetWarnings()[0].Err
require.Equal(t, "[variable:1292]Truncated incorrect tidb_cb_pd_metadata_error_rate_threshold_ratio value: '-1'", warn.Error())

// Too high, will get lowered to the max value
val, err = sv.Validate(vars, "1.1", ScopeGlobal)
require.NoError(t, err)
require.Equal(t, strconv.FormatUint(GetSysVar(TiDBCircuitBreakerPDMetadataErrorRateThresholdRatio).MaxValue, 10), val)
warn = vars.StmtCtx.GetWarnings()[1].Err
require.Equal(t, "[variable:1292]Truncated incorrect tidb_cb_pd_metadata_error_rate_threshold_ratio value: '1.1'", warn.Error())

// valid
val, err = sv.Validate(vars, "0.9", ScopeGlobal)
require.NoError(t, err)
require.Equal(t, "0.9", val)

val, err = sv.Validate(vars, "0.0", ScopeGlobal)
require.NoError(t, err)
require.Equal(t, "0.0", val)
}

func TestEnableWindowFunction(t *testing.T) {
vars := NewSessionVars(nil)
require.Equal(t, vars.EnableWindowFunction, DefEnableWindowFunction)
Expand Down
9 changes: 8 additions & 1 deletion pkg/sessionctx/variable/tidb_vars.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1250,6 +1250,9 @@ const (
// TiDBAdvancerCheckPointLagLimit controls the maximum lag could be tolerated for the checkpoint lag.
// The log backup task will be paused if the checkpoint lag is larger than it.
TiDBAdvancerCheckPointLagLimit = "tidb_advancer_check_point_lag_limit"
// TiDBCircuitBreakerPDMetadataErrorRateThresholdRatio variable is used to set ratio of errors to trip the circuit breaker for get region calls to PD
// https://github.com/tikv/rfcs/blob/master/text/0115-circuit-breaker.md
TiDBCircuitBreakerPDMetadataErrorRateThresholdRatio = "tidb_cb_pd_metadata_error_rate_threshold_ratio"
)

// TiDB intentional limits
Expand Down Expand Up @@ -1622,6 +1625,7 @@ const (
DefTiDBTSOClientRPCMode = TSOClientRPCModeDefault
DefTiDBLoadBindingTimeout = 200
DefTiDBAdvancerCheckPointLagLimit = 48 * time.Hour
DefTiDBCircuitBreakerPDMetaErrorRateRatio = 0.0
)

// Process global variables.
Expand Down Expand Up @@ -1745,7 +1749,8 @@ var (
SchemaCacheSize = atomic.NewUint64(DefTiDBSchemaCacheSize)
SchemaCacheSizeOriginText = atomic.NewString(strconv.Itoa(DefTiDBSchemaCacheSize))

AdvancerCheckPointLagLimit = atomic.NewDuration(DefTiDBAdvancerCheckPointLagLimit)
AdvancerCheckPointLagLimit = atomic.NewDuration(DefTiDBAdvancerCheckPointLagLimit)
CircuitBreakerPDMetadataErrorRateThresholdRatio = atomic.NewFloat64(0.0)
)

var (
Expand Down Expand Up @@ -1781,6 +1786,8 @@ var (
EnableStatsOwner func() error = nil
// DisableStatsOwner is the func registered by stats to disable running stats in this instance.
DisableStatsOwner func() error = nil
// ChangePDMetadataCircuitBreakerErrorRateThresholdRatio changes the error rate threshold of the PD metadata circuit breaker.
ChangePDMetadataCircuitBreakerErrorRateThresholdRatio func(uint32) = nil
)

// Hooks functions for Cluster Resource Control.
Expand Down