[utils/bloom] Optionally Update Bloom Filter Size on Reset

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/DataDog/zstd v1.5.2
 	github.com/Microsoft/go-winio v0.5.2
 	github.com/NYTimes/gziphandler v1.1.1
-	github.com/ava-labs/coreth v0.12.10-rc.4
+	github.com/ava-labs/coreth v0.12.9-rc.9.0.20240106001443-afa748159ba2
 	github.com/ava-labs/ledger-avalanche/go v0.0.0-20231102202641-ae2ebdaeac34
 	github.com/btcsuite/btcd/btcutil v1.1.3
 	github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811
@@ -24,7 +24,6 @@ require (
 	github.com/gorilla/rpc v1.2.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
-	github.com/holiman/bloomfilter/v2 v2.0.3
 	github.com/huin/goupnp v1.0.3
 	github.com/jackpal/gateway v1.0.6
 	github.com/jackpal/go-nat-pmp v1.0.2
@@ -39,7 +38,6 @@ require (
 	github.com/prometheus/client_model v0.3.0
 	github.com/rs/cors v1.7.0
 	github.com/shirou/gopsutil v3.21.11+incompatible
-	github.com/spaolacci/murmur3 v1.1.0
 	github.com/spf13/cast v1.5.0
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
@@ -109,7 +107,7 @@ require (
 	github.com/hashicorp/go-bexpr v0.1.10 // indirect
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/holiman/big v0.0.0-20221017200358-a027dc42d04e // indirect
+	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
 	github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/klauspost/compress v1.15.15 // indirect

go.sum

@@ -66,8 +66,8 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/ava-labs/coreth v0.12.10-rc.4 h1:+Ll3cpi3tZbw37lTa+1a/VnPa7xZktpbAFiuKtDKnIE=
-github.com/ava-labs/coreth v0.12.10-rc.4/go.mod h1:8pt5LW6MP/luIdhQA+gvs8LobKE8tP/5APXUiFbfb2c=
+github.com/ava-labs/coreth v0.12.9-rc.9.0.20240106001443-afa748159ba2 h1:Cnz0ekUR9GG2BI/oJPxO6qy1GM7QU59N82rCv3Q5WWU=
+github.com/ava-labs/coreth v0.12.9-rc.9.0.20240106001443-afa748159ba2/go.mod h1:YHkiBsAxZKgChyWdoeY5uMAhw+eqAfiT+m0oMwHPHuY=
 github.com/ava-labs/ledger-avalanche/go v0.0.0-20231102202641-ae2ebdaeac34 h1:mg9Uw6oZFJKytJxgxnl3uxZOs/SB8CVHg6Io4Tf99Zc=
 github.com/ava-labs/ledger-avalanche/go v0.0.0-20231102202641-ae2ebdaeac34/go.mod h1:pJxaT9bUgeRNVmNRgtCHb7sFDIRKy7CzTQVi8gGNT6g=
 github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
@@ -353,8 +353,6 @@ github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuW
 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/holiman/big v0.0.0-20221017200358-a027dc42d04e h1:pIYdhNkDh+YENVNi3gto8n9hAmRxKxoar0iE6BLucjw=
-github.com/holiman/big v0.0.0-20221017200358-a027dc42d04e/go.mod h1:j9cQbcqHQujT0oKJ38PylVfqohClLr3CvDC+Qcg+lhU=
 github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao=
 github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
 github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8=
@@ -554,8 +552,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
-github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo=
 github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=

network/p2p/gossip/bloom.go

@@ -5,40 +5,51 @@ package gossip
 
 import (
 	"crypto/rand"
-	"encoding/binary"
-	"hash"
-
-	bloomfilter "github.com/holiman/bloomfilter/v2"
 
 	"github.com/ava-labs/avalanchego/ids"
+	"github.com/ava-labs/avalanchego/utils/bloom"
+	"github.com/ava-labs/avalanchego/utils/math"
 )
 
-var _ hash.Hash64 = (*hasher)(nil)
-
-// NewBloomFilter returns a new instance of a bloom filter with at most
-// [maxExpectedElements] elements anticipated at any moment, and a false
-// positive probability of [falsePositiveProbability].
+// NewBloomFilter returns a new instance of a bloom filter with at least [minTargetElements] elements
+// anticipated at any moment, and a false positive probability of [targetFalsePositiveProbability]. If the
+// false positive probability exceeds [resetFalsePositiveProbability], the bloom filter will be reset.
+//
+// Invariant: The returned bloom filter is not safe to reset concurrently with
+// other operations. However, it is otherwise safe to access concurrently.
 func NewBloomFilter(
-	maxExpectedElements uint64,
-	falsePositiveProbability float64,
+	minTargetElements int,
+	targetFalsePositiveProbability,
+	resetFalsePositiveProbability float64,
 ) (*BloomFilter, error) {
-	bloom, err := bloomfilter.NewOptimal(
-		maxExpectedElements,
-		falsePositiveProbability,
+	numHashes, numEntries := bloom.OptimalParameters(
+		minTargetElements,
+		targetFalsePositiveProbability,
 	)
+	b, err := bloom.New(numHashes, numEntries)
 	if err != nil {
 		return nil, err
 	}
 
 	salt, err := randomSalt()
 	return &BloomFilter{
-		bloom: bloom,
-		salt:  salt,
+		minTargetElements:              minTargetElements,
+		targetFalsePositiveProbability: targetFalsePositiveProbability,
+		resetFalsePositiveProbability:  resetFalsePositiveProbability,
+
+		maxCount: bloom.EstimateCount(numHashes, numEntries, resetFalsePositiveProbability),
+		bloom:    b,
+		salt:     salt,
 	}, err
 }
 
 type BloomFilter struct {
-	bloom *bloomfilter.Filter
+	minTargetElements              int
+	targetFalsePositiveProbability float64
+	resetFalsePositiveProbability  float64
+
+	maxCount int
+	bloom    *bloom.Filter
 	// salt is provided to eventually unblock collisions in Bloom. It's possible
 	// that conflicting Gossipable items collide in the bloom filter, so a salt
 	// is generated to eventually resolve collisions.
@@ -47,49 +58,50 @@ type BloomFilter struct {
 
 func (b *BloomFilter) Add(gossipable Gossipable) {
 	h := gossipable.GossipID()
-	salted := &hasher{
-		hash: h[:],
-		salt: b.salt,
-	}
-	b.bloom.Add(salted)
+	bloom.Add(b.bloom, h[:], b.salt[:])
 }
 
 func (b *BloomFilter) Has(gossipable Gossipable) bool {
 	h := gossipable.GossipID()
-	salted := &hasher{
-		hash: h[:],
-		salt: b.salt,
-	}
-	return b.bloom.Contains(salted)
+	return bloom.Contains(b.bloom, h[:], b.salt[:])
 }
 
+// TODO: Remove error from the return
 func (b *BloomFilter) Marshal() ([]byte, []byte, error) {
-	bloomBytes, err := b.bloom.MarshalBinary()
+	bloomBytes := b.bloom.Marshal()
 	// salt must be copied here to ensure the bytes aren't overwritten if salt
 	// is later modified.
 	salt := b.salt
-	return bloomBytes, salt[:], err
+	return bloomBytes, salt[:], nil
 }
 
-// ResetBloomFilterIfNeeded resets a bloom filter if it breaches a target false
-// positive probability. Returns true if the bloom filter was reset.
+// ResetBloomFilterIfNeeded resets a bloom filter if it breaches [targetFalsePositiveProbability].
+//
+// If [targetElements] exceeds [minTargetElements], the size of the bloom filter will grow to maintain
+// the same [targetFalsePositiveProbability].
+//
+// Returns true if the bloom filter was reset.
 func ResetBloomFilterIfNeeded(
 	bloomFilter *BloomFilter,
-	falsePositiveProbability float64,
+	targetElements int,
 ) (bool, error) {
-	if bloomFilter.bloom.FalsePosititveProbability() < falsePositiveProbability {
+	if bloomFilter.bloom.Count() < bloomFilter.maxCount {
 		return false, nil
 	}
 
-	newBloom, err := bloomfilter.New(bloomFilter.bloom.M(), bloomFilter.bloom.K())
+	numHashes, numEntries := bloom.OptimalParameters(
+		math.Max(bloomFilter.minTargetElements, targetElements),
+		bloomFilter.targetFalsePositiveProbability,
+	)
+	newBloom, err := bloom.New(numHashes, numEntries)
 	if err != nil {
 		return false, err
 	}
 	salt, err := randomSalt()
 	if err != nil {
 		return false, err
 	}
-
+	bloomFilter.maxCount = bloom.EstimateCount(numHashes, numEntries, bloomFilter.resetFalsePositiveProbability)
 	bloomFilter.bloom = newBloom
 	bloomFilter.salt = salt
 	return true, nil
@@ -100,39 +112,3 @@ func randomSalt() (ids.ID, error) {
 	_, err := rand.Read(salt[:])
 	return salt, err
 }
-
-type hasher struct {
-	hash []byte
-	salt ids.ID
-}
-
-func (h *hasher) Write(p []byte) (n int, err error) {
-	h.hash = append(h.hash, p...)
-	return len(p), nil
-}
-
-func (h *hasher) Sum(b []byte) []byte {
-	h.hash = append(h.hash, b...)
-	return h.hash
-}
-
-func (h *hasher) Reset() {
-	h.hash = ids.Empty[:]
-}
-
-func (*hasher) BlockSize() int {
-	return ids.IDLen
-}
-
-func (h *hasher) Sum64() uint64 {
-	salted := ids.ID{}
-	for i := 0; i < len(h.hash) && i < ids.IDLen; i++ {
-		salted[i] = h.hash[i] ^ h.salt[i]
-	}
-
-	return binary.BigEndian.Uint64(salted[:])
-}
-
-func (h *hasher) Size() int {
-	return len(h.hash)
-}

network/p2p/gossip/bloom_test.go

@@ -6,25 +6,24 @@ package gossip
 import (
 	"testing"
 
-	bloomfilter "github.com/holiman/bloomfilter/v2"
-
 	"github.com/stretchr/testify/require"
 
 	"golang.org/x/exp/slices"
 
 	"github.com/ava-labs/avalanchego/ids"
+	"github.com/ava-labs/avalanchego/utils/bloom"
 )
 
 func TestBloomFilterRefresh(t *testing.T) {
 	tests := []struct {
-		name                     string
-		falsePositiveProbability float64
-		add                      []*testTx
-		expected                 []*testTx
+		name                          string
+		resetFalsePositiveProbability float64
+		add                           []*testTx
+		expected                      []*testTx
 	}{
 		{
-			name:                     "no refresh",
-			falsePositiveProbability: 1,
+			name:                          "no refresh",
+			resetFalsePositiveProbability: 1,
 			add: []*testTx{
 				{id: ids.ID{0}},
 			},
@@ -33,8 +32,8 @@ func TestBloomFilterRefresh(t *testing.T) {
 			},
 		},
 		{
-			name:                     "refresh",
-			falsePositiveProbability: 0.1,
+			name:                          "refresh",
+			resetFalsePositiveProbability: 0.1,
 			add: []*testTx{
 				{id: ids.ID{0}},
 				{id: ids.ID{1}},
@@ -48,10 +47,12 @@ func TestBloomFilterRefresh(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			require := require.New(t)
-			b, err := bloomfilter.New(10, 1)
+			b, err := bloom.New(1, 10)
 			require.NoError(err)
 			bloom := BloomFilter{
-				bloom: b,
+				bloom:                          b,
+				targetFalsePositiveProbability: tt.resetFalsePositiveProbability,
+				resetFalsePositiveProbability:  tt.resetFalsePositiveProbability,
 			}
 
 			for _, item := range tt.add {
@@ -61,16 +62,14 @@ func TestBloomFilterRefresh(t *testing.T) {
 				initialBloomBytes := slices.Clone(bloomBytes)
 				initialSaltBytes := slices.Clone(saltBytes)
 
-				_, err = ResetBloomFilterIfNeeded(&bloom, tt.falsePositiveProbability)
+				_, err = ResetBloomFilterIfNeeded(&bloom, len(tt.add))
 				require.NoError(err)
 				bloom.Add(item)
 
 				require.Equal(initialBloomBytes, bloomBytes)
 				require.Equal(initialSaltBytes, saltBytes)
 			}
 
-			require.Equal(uint64(len(tt.expected)), bloom.bloom.N())
-
 			for _, expected := range tt.expected {
 				require.True(bloom.Has(expected))
 			}

network/p2p/gossip/gossip_test.go

@@ -111,7 +111,7 @@ func TestGossiperGossip(t *testing.T) {
 			responseNetwork, err := p2p.NewNetwork(logging.NoLog{}, responseSender, prometheus.NewRegistry(), "")
 			require.NoError(err)
 
-			responseBloom, err := NewBloomFilter(1000, 0.01)
+			responseBloom, err := NewBloomFilter(1000, 0.01, 0.05)
 			require.NoError(err)
 			responseSet := &testSet{
 				txs:   make(map[ids.ID]*testTx),
@@ -143,7 +143,7 @@ func TestGossiperGossip(t *testing.T) {
 			require.NoError(err)
 			require.NoError(requestNetwork.Connected(context.Background(), ids.EmptyNodeID, nil))
 
-			bloom, err := NewBloomFilter(1000, 0.01)
+			bloom, err := NewBloomFilter(1000, 0.01, 0.05)
 			require.NoError(err)
 			requestSet := &testSet{
 				txs:   make(map[ids.ID]*testTx),
@@ -365,7 +365,7 @@ func TestPushGossipE2E(t *testing.T) {
 	knownTx := &testTx{id: ids.GenerateTestID()}
 
 	log := logging.NoLog{}
-	bloom, err := NewBloomFilter(100, 0.01)
+	bloom, err := NewBloomFilter(100, 0.01, 0.05)
 	require.NoError(err)
 	set := &testSet{
 		txs:   make(map[ids.ID]*testTx),

network/p2p/gossip/handler.go

@@ -8,15 +8,14 @@ import (
 	"fmt"
 	"time"
 
-	bloomfilter "github.com/holiman/bloomfilter/v2"
-
 	"go.uber.org/zap"
 
 	"google.golang.org/protobuf/proto"
 
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/network/p2p"
 	"github.com/ava-labs/avalanchego/proto/pb/sdk"
+	"github.com/ava-labs/avalanchego/utils/bloom"
 	"github.com/ava-labs/avalanchego/utils/logging"
 )
 
@@ -62,19 +61,18 @@ func (h Handler[T]) AppRequest(_ context.Context, _ ids.NodeID, _ time.Time, req
 		return nil, err
 	}
 
-	filter := &BloomFilter{
-		bloom: &bloomfilter.Filter{},
-		salt:  salt,
-	}
-	if err := filter.bloom.UnmarshalBinary(request.Filter); err != nil {
+	filter, err := bloom.Parse(request.Filter)
+	if err != nil {
 		return nil, err
 	}
 
 	responseSize := 0
 	gossipBytes := make([][]byte, 0)
 	h.set.Iterate(func(gossipable T) bool {
+		gossipID := gossipable.GossipID()
+
 		// filter out what the requesting peer already knows about
-		if filter.Has(gossipable) {
+		if bloom.Contains(filter, gossipID[:], salt[:]) {
 			return true
 		}