Skip to content

ledger: add pebbledb support #5606

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
algorandskiy merged 10 commits into from
Aug 9, 2023
Merged

ledger: add pebbledb support #5606

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
57afc2e
feat: add pebbledb support
icorderi Jul 25, 2023
af13664
test: `LookupOnlineRoundParams`
icorderi Jul 25, 2023
d9cc3bc
misc: make tidy
icorderi Jul 25, 2023
e6f2f18
fix typo
icorderi Jul 25, 2023
3ffeb21
Merge branch 'master' into feat/kv/pr-4
icorderi Aug 7, 2023
3d77bb0
misc: make tidy
icorderi Aug 7, 2023
f072b0d
misc: add some comments warning of perf impact of changing some values
icorderi Aug 7, 2023
b17cb7c
pr comments
icorderi Aug 7, 2023
3d92224
FD limits check for pebbledb +1000 open descriptiors
algorandskiy Aug 8, 2023
a782c10
Update daemon/algod/server.go
algorandskiy Aug 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions daemon/algod/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,20 @@ func (s *Server) Initialize(cfg config.Local, phonebookAddresses []string, genes
if err != nil {
return fmt.Errorf("Initialize() err: %w", err)
}
// TODO: remove this after making pebble support official
// and integrate the value into ReservedFDs config parameter.
if cfg.StorageEngine == "pebbledb" {
fdRequired = ot.Add(fdRequired, 1000)
if ot.Overflowed {
return errors.New(
"Initialize() overflowed when adding up fdRequired and 1000 needed for pebbledb")
}
err = util.SetFdSoftLimit(fdRequired)
if err != nil {
return fmt.Errorf("Initialize() failed to set FD limit for pebbledb backend, err: %w", err)
}
}

if cfg.IsGossipServer() {
var ot basics.OverflowTracker
fdRequired = ot.Add(fdRequired, uint64(cfg.IncomingConnectionsLimit))
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ require (
github.com/algorand/sortition v1.0.0
github.com/algorand/websocket v1.4.6
github.com/aws/aws-sdk-go v1.33.0
github.com/cockroachdb/pebble v0.0.0-20230807162746-af8c5f279001
github.com/consensys/gnark-crypto v0.7.0
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c
github.com/dchest/siphash v1.2.1
Expand Down Expand Up @@ -52,9 +53,9 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cockroachdb/errors v1.8.1 // indirect
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
github.com/cockroachdb/pebble v0.0.0-20230227185959-8285e8dd5c08 // indirect
github.com/cockroachdb/redact v1.0.8 // indirect
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
github.com/cockroachdb/tokenbucket v0.0.0-20230613231145-182959a1fad6 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
Expand All @@ -64,7 +65,6 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
Expand Down
11 changes: 6 additions & 5 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,18 +132,20 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4=
github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA=
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM=
github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y=
github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY=
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
github.com/cockroachdb/pebble v0.0.0-20230227185959-8285e8dd5c08 h1:acwj2RLsc4NK34fzBUQp5oy4oN+XTbRsEZVCX4puE+U=
github.com/cockroachdb/pebble v0.0.0-20230227185959-8285e8dd5c08/go.mod h1:9lRMC4XN3/BLPtIp6kAKwIaHu369NOf2rMucPzipz50=
github.com/cockroachdb/pebble v0.0.0-20230807162746-af8c5f279001 h1:+woucWZrHqZbpUr61pxhUnF7NSmUmrg9zT0z3Xz8U1w=
github.com/cockroachdb/pebble v0.0.0-20230807162746-af8c5f279001/go.mod h1:FN5O47SBEz5+kO9fG8UTR64g2WS1u5ZFCgTvxGjoSks=
github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw=
github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM=
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ=
github.com/cockroachdb/tokenbucket v0.0.0-20230613231145-182959a1fad6 h1:DJK8W/iB+s/qkTtmXSrHA49lp5O3OsR7E6z4byOLy34=
github.com/cockroachdb/tokenbucket v0.0.0-20230613231145-182959a1fad6/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
github.com/consensys/gnark-crypto v0.7.0 h1:rwdy8+ssmLYRqKp+ryRRgQJl/rCq2uv+n83cOydm5UE=
github.com/consensys/gnark-crypto v0.7.0/go.mod h1:KPSuJzyxkJA8xZ/+CV47tyqkr9MmpZA3PXivK4VPrVg=
Expand Down Expand Up @@ -293,8 +295,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
Expand Down
6 changes: 5 additions & 1 deletion ledger/ledger.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
"github.com/algorand/go-algorand/ledger/ledgercore"
"github.com/algorand/go-algorand/ledger/store/blockdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb"
"github.com/algorand/go-algorand/ledger/store/trackerdb/pebbledbdriver"
"github.com/algorand/go-algorand/ledger/store/trackerdb/sqlitedriver"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/protocol"
Expand Down Expand Up @@ -296,9 +297,12 @@ func openLedgerDB(dbPathPrefix string, dbMem bool, cfg config.Local, log logging
go func() {
var lerr error
switch cfg.StorageEngine {
case "pebbledb":
dir := dbPathPrefix + "/tracker.pebble"
trackerDBs, lerr = pebbledbdriver.Open(dir, dbMem, config.Consensus[protocol.ConsensusCurrentVersion], log)
Comment thread
AlgoAxel marked this conversation as resolved.
// anything else will initialize a sqlite engine.
case "sqlite":
fallthrough
// anything else will initialize a sqlite engine.
default:
file := dbPathPrefix + ".tracker.sqlite"
trackerDBs, lerr = sqlitedriver.Open(file, dbMem, log)
Expand Down
Loading