From 7dfd0f5b4389cd2665b6203e19b7aa1bfe4d5e71 Mon Sep 17 00:00:00 2001 From: Miguel Filipe Date: Thu, 25 Jul 2024 07:32:19 -0700 Subject: [PATCH] tweak the in-mem buffer for more stable throughput --- ingester/mainloop.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ingester/mainloop.go b/ingester/mainloop.go index ee2629d..a91c448 100644 --- a/ingester/mainloop.go +++ b/ingester/mainloop.go @@ -46,8 +46,10 @@ func (i *ingester) Run(ctx context.Context, startBlockNumber int64, maxCount int defer close(blockNumbers) // We buffer the block channel so that RPC requests can be made concurrently with sending blocks to Dune. - // We limit the buffer size to the same number of concurrent requests, so we exert some backpressure. - blocks := make(chan models.RPCBlock, maxBatchSize) + // We limit the buffer size to the k * maxBatchSize, + // so we exert some backpressure. + // but having enough to feed full batches to DuneAPI + blocks := make(chan models.RPCBlock, 2*maxBatchSize) defer close(blocks) // Start MaxBatchSize goroutines to consume blocks concurrently