diff --git a/go/pools/resource_pool.go b/go/pools/resource_pool.go index 8e92f0bf5ee..945393c792e 100644 --- a/go/pools/resource_pool.go +++ b/go/pools/resource_pool.go @@ -60,6 +60,7 @@ type ResourcePool struct { waitCount sync2.AtomicInt64 waitTime sync2.AtomicDuration idleClosed sync2.AtomicInt64 + exhausted sync2.AtomicInt64 capacity sync2.AtomicInt64 idleTimeout sync2.AtomicDuration @@ -230,7 +231,9 @@ func (rp *ResourcePool) get(ctx context.Context) (resource Resource, err error) } rp.active.Add(1) } - rp.available.Add(-1) + if rp.available.Add(-1) <= 0 { + rp.exhausted.Add(1) + } rp.inUse.Add(1) return wrapper.resource, err } @@ -334,7 +337,7 @@ func (rp *ResourcePool) SetIdleTimeout(idleTimeout time.Duration) { // StatsJSON returns the stats in JSON format. func (rp *ResourcePool) StatsJSON() string { - return fmt.Sprintf(`{"Capacity": %v, "Available": %v, "Active": %v, "InUse": %v, "MaxCapacity": %v, "WaitCount": %v, "WaitTime": %v, "IdleTimeout": %v, "IdleClosed": %v}`, + return fmt.Sprintf(`{"Capacity": %v, "Available": %v, "Active": %v, "InUse": %v, "MaxCapacity": %v, "WaitCount": %v, "WaitTime": %v, "IdleTimeout": %v, "IdleClosed": %v, "Exhausted": %v}`, rp.Capacity(), rp.Available(), rp.Active(), @@ -344,6 +347,7 @@ func (rp *ResourcePool) StatsJSON() string { rp.WaitTime().Nanoseconds(), rp.IdleTimeout().Nanoseconds(), rp.IdleClosed(), + rp.Exhausted(), ) } @@ -392,3 +396,8 @@ func (rp *ResourcePool) IdleTimeout() time.Duration { func (rp *ResourcePool) IdleClosed() int64 { return rp.idleClosed.Get() } + +// Exhausted returns the number of times Available dropped below 1 +func (rp *ResourcePool) Exhausted() int64 { + return rp.exhausted.Get() +} diff --git a/go/pools/resource_pool_test.go b/go/pools/resource_pool_test.go index 15c21c9202e..f9a7c9ac3a7 100644 --- a/go/pools/resource_pool_test.go +++ b/go/pools/resource_pool_test.go @@ -247,7 +247,7 @@ func TestShrinking(t *testing.T) { p.SetCapacity(3) done <- true }() - expected := `{"Capacity": 3, "Available": 0, "Active": 4, "InUse": 4, "MaxCapacity": 5, "WaitCount": 0, "WaitTime": 0, "IdleTimeout": 1000000000, "IdleClosed": 0}` + expected := `{"Capacity": 3, "Available": 0, "Active": 4, "InUse": 4, "MaxCapacity": 5, "WaitCount": 0, "WaitTime": 0, "IdleTimeout": 1000000000, "IdleClosed": 0, "Exhausted": 0}` for i := 0; i < 10; i++ { time.Sleep(10 * time.Millisecond) stats := p.StatsJSON() @@ -266,7 +266,7 @@ func TestShrinking(t *testing.T) { p.Put(resources[i]) } stats := p.StatsJSON() - expected = `{"Capacity": 3, "Available": 3, "Active": 3, "InUse": 0, "MaxCapacity": 5, "WaitCount": 0, "WaitTime": 0, "IdleTimeout": 1000000000, "IdleClosed": 0}` + expected = `{"Capacity": 3, "Available": 3, "Active": 3, "InUse": 0, "MaxCapacity": 5, "WaitCount": 0, "WaitTime": 0, "IdleTimeout": 1000000000, "IdleClosed": 0, "Exhausted": 0}` if stats != expected { t.Errorf(`expecting '%s', received '%s'`, expected, stats) } @@ -389,7 +389,7 @@ func TestClosing(t *testing.T) { // Wait for goroutine to call Close time.Sleep(10 * time.Millisecond) stats := p.StatsJSON() - expected := `{"Capacity": 0, "Available": 0, "Active": 5, "InUse": 5, "MaxCapacity": 5, "WaitCount": 0, "WaitTime": 0, "IdleTimeout": 1000000000, "IdleClosed": 0}` + expected := `{"Capacity": 0, "Available": 0, "Active": 5, "InUse": 5, "MaxCapacity": 5, "WaitCount": 0, "WaitTime": 0, "IdleTimeout": 1000000000, "IdleClosed": 0, "Exhausted": 1}` if stats != expected { t.Errorf(`expecting '%s', received '%s'`, expected, stats) } @@ -409,7 +409,7 @@ func TestClosing(t *testing.T) { } stats = p.StatsJSON() - expected = `{"Capacity": 0, "Available": 0, "Active": 0, "InUse": 0, "MaxCapacity": 5, "WaitCount": 0, "WaitTime": 0, "IdleTimeout": 1000000000, "IdleClosed": 0}` + expected = `{"Capacity": 0, "Available": 0, "Active": 0, "InUse": 0, "MaxCapacity": 5, "WaitCount": 0, "WaitTime": 0, "IdleTimeout": 1000000000, "IdleClosed": 0, "Exhausted": 1}` if stats != expected { t.Errorf(`expecting '%s', received '%s'`, expected, stats) } @@ -563,7 +563,7 @@ func TestCreateFail(t *testing.T) { t.Errorf("Expecting Failed, received %v", err) } stats := p.StatsJSON() - expected := `{"Capacity": 5, "Available": 5, "Active": 0, "InUse": 0, "MaxCapacity": 5, "WaitCount": 0, "WaitTime": 0, "IdleTimeout": 1000000000, "IdleClosed": 0}` + expected := `{"Capacity": 5, "Available": 5, "Active": 0, "InUse": 0, "MaxCapacity": 5, "WaitCount": 0, "WaitTime": 0, "IdleTimeout": 1000000000, "IdleClosed": 0, "Exhausted": 0}` if stats != expected { t.Errorf(`expecting '%s', received '%s'`, expected, stats) } diff --git a/go/vt/dbconnpool/connection_pool.go b/go/vt/dbconnpool/connection_pool.go index eedee5ddff9..7480540a691 100644 --- a/go/vt/dbconnpool/connection_pool.go +++ b/go/vt/dbconnpool/connection_pool.go @@ -84,6 +84,7 @@ func NewConnectionPool(name string, capacity int, idleTimeout time.Duration, dns stats.NewCounterDurationFunc(name+"WaitTime", "Connection pool wait time", cp.WaitTime) stats.NewGaugeDurationFunc(name+"IdleTimeout", "Connection pool idle timeout", cp.IdleTimeout) stats.NewGaugeFunc(name+"IdleClosed", "Connection pool idle closed", cp.IdleClosed) + stats.NewCounterFunc(name+"Exhausted", "Number of times pool had zero available slots", cp.Exhausted) return cp } @@ -355,3 +356,12 @@ func (cp *ConnectionPool) IdleClosed() int64 { } return p.IdleClosed() } + +// Exhausted returns the number of times available went to zero for the pool. +func (cp *ConnectionPool) Exhausted() int64 { + p := cp.pool() + if p == nil { + return 0 + } + return p.Exhausted() +} diff --git a/go/vt/vttablet/tabletserver/connpool/pool.go b/go/vt/vttablet/tabletserver/connpool/pool.go index eee178b36f7..ed2fbc9e745 100644 --- a/go/vt/vttablet/tabletserver/connpool/pool.go +++ b/go/vt/vttablet/tabletserver/connpool/pool.go @@ -98,6 +98,7 @@ func New( stats.NewCounterDurationFunc(name+"WaitTime", "Tablet server wait time", cp.WaitTime) stats.NewGaugeDurationFunc(name+"IdleTimeout", "Tablet server idle timeout", cp.IdleTimeout) stats.NewCounterFunc(name+"IdleClosed", "Tablet server conn pool idle closed", cp.IdleClosed) + stats.NewCounterFunc(name+"Exhausted", "Number of times pool had zero available slots", cp.Exhausted) return cp } @@ -296,6 +297,15 @@ func (cp *Pool) IdleClosed() int64 { return p.IdleClosed() } +// Exhausted returns the number of times available went to zero for the pool. +func (cp *Pool) Exhausted() int64 { + p := cp.pool() + if p == nil { + return 0 + } + return p.Exhausted() +} + func (cp *Pool) isCallerIDAppDebug(ctx context.Context) bool { if cp.appDebugParams == nil || cp.appDebugParams.Uname == "" { return false