etcdserver: rate limit lease revoke per second

Instead of having maximum number of inflight revoke
routine, rate limit revokes 'per second'.

Fix #8097.

Signed-off-by: Gyu-Ho Lee <[email protected]>

etcdserver/server.go

@@ -54,9 +54,11 @@ import (
 	"github.com/coreos/etcd/store"
 	"github.com/coreos/etcd/version"
 	"github.com/coreos/etcd/wal"
+
 	"github.com/coreos/go-semver/semver"
 	"github.com/coreos/pkg/capnslog"
 	"golang.org/x/net/context"
+	"golang.org/x/time/rate"
 )
 
 const (
@@ -81,8 +83,9 @@ const (
 
 	releaseDelayAfterSnapshot = 30 * time.Second
 
-	// maxPendingRevokes is the maximum number of outstanding expired lease revocations.
-	maxPendingRevokes = 16
+	// TODO: make this configurable?
+	// maximum number of leases to revoke per second
+	leaseRevokeRate = 1000
 
 	recommendedMaxRequestBytes = 10 * 1024 * 1024
 )
@@ -730,6 +733,7 @@ func (s *EtcdServer) run() {
 		expiredLeaseC = s.lessor.ExpiredLeasesC()
 	}
 
+	revokerLimiter := rate.NewLimiter(rate.Limit(leaseRevokeRate), leaseRevokeRate)
 	for {
 		select {
 		case ap := <-s.r.apply():
@@ -738,19 +742,18 @@ func (s *EtcdServer) run() {
 		case leases := <-expiredLeaseC:
 			s.goAttach(func() {
 				// Increases throughput of expired leases deletion process through parallelization
-				c := make(chan struct{}, maxPendingRevokes)
 				for _, lease := range leases {
 					select {
-					case c <- struct{}{}:
 					case <-s.stopping:
 						return
+					default:
 					}
+					revokerLimiter.Wait(context.Background())
 					lid := lease.ID
 					s.goAttach(func() {
 						ctx := s.authStore.WithRoot(s.ctx)
 						s.LeaseRevoke(ctx, &pb.LeaseRevokeRequest{ID: int64(lid)})
 						leaseExpired.Inc()
-						<-c
 					})
 				}
 			})