Disable excessive CPU rate limiting

.github/workflows/build.yml

@@ -16,7 +16,10 @@ permissions:
 
 jobs:
   goreleaser:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, ubuntu-20.04]
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -35,6 +38,12 @@ jobs:
           # TODO: automate github release page announce and artifact uploads
           # https://goreleaser.com/cmd/goreleaser_release/
           args: release --rm-dist --skip-announce --skip-publish
+      - name: Archive Builds
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist_${{ matrix.os }}
+          path: ./dist
+
 
         # to automate release announcement
         # https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret

go.mod

@@ -58,8 +58,8 @@ require (
 	golang.org/x/term v0.7.0
 	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af
 	gonum.org/v1/gonum v0.11.0
-	google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c
-	google.golang.org/grpc v1.50.1
+	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f
+	google.golang.org/grpc v1.54.0
 	google.golang.org/protobuf v1.30.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
@@ -138,9 +138,9 @@ require (
 	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -703,8 +703,8 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
 golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -807,8 +807,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -943,8 +943,8 @@ google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
 google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c h1:QgY/XxIAIeccR+Ca/rDdKubLIU9rcJ3xfy1DC/Wd2Oo=
-google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo=
+google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
+google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
 google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -967,8 +967,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY=
-google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI=
+google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag=
+google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

network/peer/message_queue.go

@@ -55,9 +55,9 @@ type MessageQueue interface {
 type throttledMessageQueue struct {
 	onFailed SendFailedCallback
 	// [id] of the peer we're sending messages to
-	id                   ids.NodeID
-	log                  logging.Logger
-	outboundMsgThrottler throttling.OutboundMsgThrottler
+	id  ids.NodeID
+	log logging.Logger
+	// outboundMsgThrottler throttling.OutboundMsgThrottler
 
 	// Signalled when a message is added to the queue and when Close() is
 	// called.
@@ -76,15 +76,15 @@ func NewThrottledMessageQueue(
 	onFailed SendFailedCallback,
 	id ids.NodeID,
 	log logging.Logger,
-	outboundMsgThrottler throttling.OutboundMsgThrottler,
+	_ throttling.OutboundMsgThrottler,
 ) MessageQueue {
 	return &throttledMessageQueue{
-		onFailed:             onFailed,
-		id:                   id,
-		log:                  log,
-		outboundMsgThrottler: outboundMsgThrottler,
-		cond:                 sync.NewCond(&sync.Mutex{}),
-		queue:                buffer.NewUnboundedDeque[message.OutboundMessage](initialQueueSize),
+		onFailed: onFailed,
+		id:       id,
+		log:      log,
+		// outboundMsgThrottler: outboundMsgThrottler,
+		cond:  sync.NewCond(&sync.Mutex{}),
+		queue: buffer.NewUnboundedDeque[message.OutboundMessage](initialQueueSize),
 	}
 }
 
@@ -101,16 +101,16 @@ func (q *throttledMessageQueue) Push(ctx context.Context, msg message.OutboundMe
 	}
 
 	// Acquire space on the outbound message queue, or drop [msg] if we can't.
-	if !q.outboundMsgThrottler.Acquire(msg, q.id) {
-		q.log.Debug(
-			"dropping outgoing message",
-			zap.String("reason", "rate-limiting"),
-			zap.Stringer("messageOp", msg.Op()),
-			zap.Stringer("nodeID", q.id),
-		)
-		q.onFailed.SendFailed(msg)
-		return false
-	}
+	// if !q.outboundMsgThrottler.Acquire(msg, q.id) {
+	// 	q.log.Debug(
+	// 		"dropping outgoing message",
+	// 		zap.String("reason", "rate-limiting"),
+	// 		zap.Stringer("messageOp", msg.Op()),
+	// 		zap.Stringer("nodeID", q.id),
+	// 	)
+	// 	q.onFailed.SendFailed(msg)
+	// 	return false
+	// }
 
 	// Invariant: must call q.outboundMsgThrottler.Release(msg, q.id) when [msg]
 	// is popped or, if this queue closes before [msg] is popped, when this
@@ -120,13 +120,13 @@ func (q *throttledMessageQueue) Push(ctx context.Context, msg message.OutboundMe
 	defer q.cond.L.Unlock()
 
 	if q.closed {
-		q.log.Debug(
+		q.log.Info(
 			"dropping outgoing message",
 			zap.String("reason", "closed queue"),
 			zap.Stringer("messageOp", msg.Op()),
 			zap.Stringer("nodeID", q.id),
 		)
-		q.outboundMsgThrottler.Release(msg, q.id)
+		// q.outboundMsgThrottler.Release(msg, q.id)
 		q.onFailed.SendFailed(msg)
 		return false
 	}
@@ -170,7 +170,7 @@ func (q *throttledMessageQueue) PopNow() (message.OutboundMessage, bool) {
 func (q *throttledMessageQueue) pop() message.OutboundMessage {
 	msg, _ := q.queue.PopLeft()
 
-	q.outboundMsgThrottler.Release(msg, q.id)
+	// q.outboundMsgThrottler.Release(msg, q.id)
 	return msg
 }
 
@@ -186,7 +186,7 @@ func (q *throttledMessageQueue) Close() {
 
 	for q.queue.Len() > 0 {
 		msg, _ := q.queue.PopLeft()
-		q.outboundMsgThrottler.Release(msg, q.id)
+		// q.outboundMsgThrottler.Release(msg, q.id)
 		q.onFailed.SendFailed(msg)
 	}
 	q.queue = nil

snow/networking/handler/message_queue.go

@@ -203,33 +203,35 @@ func (m *messageQueue) Shutdown() {
 }
 
 // canPop will return true for at least one message in [m.msgs]
-func (m *messageQueue) canPop(msg message.InboundMessage) bool {
-	// Always pop connected and disconnected messages.
-	if op := msg.Op(); op == message.ConnectedOp || op == message.DisconnectedOp || op == message.ConnectedSubnetOp {
-		return true
-	}
-
-	// If the deadline to handle [msg] has passed, always pop it.
-	// It will be dropped immediately.
-	if expiration := msg.Expiration(); m.clock.Time().After(expiration) {
-		return true
-	}
-	// Every node has some allowed CPU allocation depending on
-	// the number of nodes with unprocessed messages.
-	baseMaxCPU := 1 / float64(len(m.nodeToUnprocessedMsgs))
-	nodeID := msg.NodeID()
-	weight := m.vdrs.GetWeight(nodeID)
-	// The sum of validator weights should never be 0, but handle
-	// that case for completeness here to avoid divide by 0.
-	portionWeight := float64(0)
-	totalVdrsWeight := m.vdrs.Weight()
-	if totalVdrsWeight != 0 {
-		portionWeight = float64(weight) / float64(totalVdrsWeight)
-	}
-	// Validators are allowed to use more CPU. More weight --> more CPU use allowed.
-	recentCPUUsage := m.cpuTracker.Usage(nodeID, m.clock.Time())
-	maxCPU := baseMaxCPU + (1.0-baseMaxCPU)*portionWeight
-	return recentCPUUsage <= maxCPU
+func (*messageQueue) canPop(msg message.InboundMessage) bool {
+	// Disable excessive CPU rate limiting
+	return true
+	// // Always pop connected and disconnected messages.
+	// if op := msg.Op(); op == message.ConnectedOp || op == message.DisconnectedOp || op == message.ConnectedSubnetOp {
+	// 	return true
+	// }
+
+	// // If the deadline to handle [msg] has passed, always pop it.
+	// // It will be dropped immediately.
+	// if expiration := msg.Expiration(); m.clock.Time().After(expiration) {
+	// 	return true
+	// }
+	// // Every node has some allowed CPU allocation depending on
+	// // the number of nodes with unprocessed messages.
+	// baseMaxCPU := 1 / float64(len(m.nodeToUnprocessedMsgs))
+	// nodeID := msg.NodeID()
+	// weight := m.vdrs.GetWeight(nodeID)
+	// // The sum of validator weights should never be 0, but handle
+	// // that case for completeness here to avoid divide by 0.
+	// portionWeight := float64(0)
+	// totalVdrsWeight := m.vdrs.Weight()
+	// if totalVdrsWeight != 0 {
+	// 	portionWeight = float64(weight) / float64(totalVdrsWeight)
+	// }
+	// // Validators are allowed to use more CPU. More weight --> more CPU use allowed.
+	// recentCPUUsage := m.cpuTracker.Usage(nodeID, m.clock.Time())
+	// maxCPU := baseMaxCPU + (1.0-baseMaxCPU)*portionWeight
+	// return recentCPUUsage <= maxCPU
 }
 
 type msgAndContext struct {

vms/rpcchainvm/grpcutils/server.go

@@ -25,21 +25,6 @@ const (
 	// of Timeout and if no activity is seen even after that the connection is
 	// closed. grpc-go default 20s
 	defaultServerKeepAliveTimeout = 20 * time.Second
-	// Duration for the maximum amount of time a http2 connection can exist
-	// before sending GOAWAY. Internally in gRPC a +-10% jitter is added to
-	// mitigate retry storms.
-	defaultServerMaxConnectionAge = 10 * time.Minute
-	// After MaxConnectionAge, MaxConnectionAgeGrace specifies the amount of time
-	// between when the server sends a GOAWAY to the client to initiate graceful
-	// shutdown, and when the server closes the connection.
-	//
-	// The server expects that this grace period will allow the client to complete
-	// any ongoing requests, after which it will forcefully terminate the connection.
-	// If a request takes longer than this grace period, it will *fail*.
-	// We *never* want an RPC to live longer than this value.
-	//
-	// invariant: Any value < 1 second will be internally overridden by gRPC.
-	defaultServerMaxConnectionAgeGrace = math.MaxInt64
 )
 
 var DefaultServerOptions = []grpc.ServerOption{
@@ -51,10 +36,8 @@ var DefaultServerOptions = []grpc.ServerOption{
 		PermitWithoutStream: defaultPermitWithoutStream,
 	}),
 	grpc.KeepaliveParams(keepalive.ServerParameters{
-		Time:                  defaultServerKeepAliveInterval,
-		Timeout:               defaultServerKeepAliveTimeout,
-		MaxConnectionAge:      defaultServerMaxConnectionAge,
-		MaxConnectionAgeGrace: defaultServerMaxConnectionAgeGrace,
+		Time:    defaultServerKeepAliveInterval,
+		Timeout: defaultServerKeepAliveTimeout,
 	}),
 }