Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add getFeeStats method #172

Merged
merged 20 commits into from
May 17, 2024
Merged

Add getFeeStats method #172

Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
90ba038
Add getFeeStats method
2opremio May 9, 2024
3452119
Merge branch 'main' into get-fee-stats
2opremio May 10, 2024
ee10615
Remove TODO
2opremio May 10, 2024
d0551b7
Fix unit tests
2opremio May 13, 2024
9aac80d
Fix typo when appending to soroban fees
2opremio May 13, 2024
2a450d2
Merge branch 'main' into get-fee-stats
2opremio May 14, 2024
04bdced
Rename count to feeCount and add ledgerCount
2opremio May 14, 2024
57ecad1
Add integration test and a few fixes
2opremio May 16, 2024
965a13f
Fix soroban inclusion fee computation
2opremio May 16, 2024
95cfc5c
Merge branch 'main' into get-fee-stats
2opremio May 16, 2024
c05251f
Enforce TransactionMetaExtV1
2opremio May 16, 2024
fc3a9c2
Merge branch 'main' into get-fee-stats
2opremio May 16, 2024
76c0237
Merge branch 'main' into get-fee-stats
2opremio May 16, 2024
0469068
Fix unit tests
2opremio May 16, 2024
d9b9bbc
Expand tests and fix mode computation
2opremio May 16, 2024
6bcf37f
Merge branch 'main' into get-fee-stats
2opremio May 16, 2024
bd0eb1d
Fix tests
2opremio May 16, 2024
c3e1eb2
Use transactionCount instead of feeCount
2opremio May 16, 2024
783a245
Extend classic window to 10 ledgers
2opremio May 17, 2024
80626b3
Merge branch 'main' into get-fee-stats
2opremio May 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 45 additions & 41 deletions cmd/soroban-rpc/internal/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,47 +20,51 @@ type Config struct {
CaptiveCoreConfigPath string
CaptiveCoreHTTPPort uint

Endpoint string
AdminEndpoint string
CheckpointFrequency uint32
CoreRequestTimeout time.Duration
DefaultEventsLimit uint
EventLedgerRetentionWindow uint32
FriendbotURL string
HistoryArchiveURLs []string
HistoryArchiveUserAgent string
IngestionTimeout time.Duration
LogFormat LogFormat
LogLevel logrus.Level
MaxEventsLimit uint
MaxHealthyLedgerLatency time.Duration
NetworkPassphrase string
PreflightWorkerCount uint
PreflightWorkerQueueSize uint
PreflightEnableDebug bool
SQLiteDBPath string
TransactionLedgerRetentionWindow uint32
RequestBacklogGlobalQueueLimit uint
RequestBacklogGetHealthQueueLimit uint
RequestBacklogGetEventsQueueLimit uint
RequestBacklogGetNetworkQueueLimit uint
RequestBacklogGetVersionInfoQueueLimit uint
RequestBacklogGetLatestLedgerQueueLimit uint
RequestBacklogGetLedgerEntriesQueueLimit uint
RequestBacklogGetTransactionQueueLimit uint
RequestBacklogSendTransactionQueueLimit uint
RequestBacklogSimulateTransactionQueueLimit uint
RequestExecutionWarningThreshold time.Duration
MaxRequestExecutionDuration time.Duration
MaxGetHealthExecutionDuration time.Duration
MaxGetEventsExecutionDuration time.Duration
MaxGetNetworkExecutionDuration time.Duration
MaxGetVersionInfoExecutionDuration time.Duration
MaxGetLatestLedgerExecutionDuration time.Duration
MaxGetLedgerEntriesExecutionDuration time.Duration
MaxGetTransactionExecutionDuration time.Duration
MaxSendTransactionExecutionDuration time.Duration
MaxSimulateTransactionExecutionDuration time.Duration
Endpoint string
AdminEndpoint string
CheckpointFrequency uint32
CoreRequestTimeout time.Duration
DefaultEventsLimit uint
EventLedgerRetentionWindow uint32
FriendbotURL string
HistoryArchiveURLs []string
HistoryArchiveUserAgent string
IngestionTimeout time.Duration
LogFormat LogFormat
LogLevel logrus.Level
MaxEventsLimit uint
MaxHealthyLedgerLatency time.Duration
NetworkPassphrase string
PreflightWorkerCount uint
PreflightWorkerQueueSize uint
PreflightEnableDebug bool
SQLiteDBPath string
TransactionLedgerRetentionWindow uint32
SorobanFeeStatsLedgerRetentionWindow uint32
ClassicFeeStatsLedgerRetentionWindow uint32
RequestBacklogGlobalQueueLimit uint
RequestBacklogGetHealthQueueLimit uint
RequestBacklogGetEventsQueueLimit uint
RequestBacklogGetNetworkQueueLimit uint
RequestBacklogGetVersionInfoQueueLimit uint
RequestBacklogGetLatestLedgerQueueLimit uint
RequestBacklogGetLedgerEntriesQueueLimit uint
RequestBacklogGetTransactionQueueLimit uint
RequestBacklogSendTransactionQueueLimit uint
RequestBacklogSimulateTransactionQueueLimit uint
RequestBacklogGetFeeStatsTransactionQueueLimit uint
RequestExecutionWarningThreshold time.Duration
MaxRequestExecutionDuration time.Duration
MaxGetHealthExecutionDuration time.Duration
MaxGetEventsExecutionDuration time.Duration
MaxGetNetworkExecutionDuration time.Duration
MaxGetVersionInfoExecutionDuration time.Duration
MaxGetLatestLedgerExecutionDuration time.Duration
MaxGetLedgerEntriesExecutionDuration time.Duration
MaxGetTransactionExecutionDuration time.Duration
MaxSendTransactionExecutionDuration time.Duration
MaxSimulateTransactionExecutionDuration time.Duration
MaxGetFeeStatsExecutionDuration time.Duration

// We memoize these, so they bind to pflags correctly
optionsCache *ConfigOptions
Expand Down
27 changes: 27 additions & 0 deletions cmd/soroban-rpc/internal/config/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,20 @@ func (cfg *Config) options() ConfigOptions {
DefaultValue: uint32(1440),
Validate: positive,
},
{
Name: "classic-fee-stats-retention-window",
Usage: "configures classic fee stats retention window expressed in number of ledgers",
ConfigKey: &cfg.ClassicFeeStatsLedgerRetentionWindow,
DefaultValue: uint32(10),
Validate: positive,
},
{
Name: "soroban-fee-stats-retention-window",
Usage: "configures soroban inclusion fee stats retention window expressed in number of ledgers",
ConfigKey: &cfg.SorobanFeeStatsLedgerRetentionWindow,
DefaultValue: uint32(50),
Validate: positive,
},
{
Name: "max-events-limit",
Usage: "Maximum amount of events allowed in a single getEvents response",
Expand Down Expand Up @@ -344,6 +358,13 @@ func (cfg *Config) options() ConfigOptions {
DefaultValue: uint(100),
Validate: positive,
},
{
TomlKey: strutils.KebabToConstantCase("request-backlog-get-fee-stats-queue-limit"),
Usage: "Maximum number of outstanding GetFeeStats requests",
ConfigKey: &cfg.RequestBacklogGetFeeStatsTransactionQueueLimit,
DefaultValue: uint(100),
Validate: positive,
},
{
TomlKey: strutils.KebabToConstantCase("request-execution-warning-threshold"),
Usage: "The request execution warning threshold is the predetermined maximum duration of time that a request can take to be processed before a warning would be generated",
Expand Down Expand Up @@ -410,6 +431,12 @@ func (cfg *Config) options() ConfigOptions {
ConfigKey: &cfg.MaxSimulateTransactionExecutionDuration,
DefaultValue: 15 * time.Second,
},
{
TomlKey: strutils.KebabToConstantCase("max-get-fee-stats-execution-duration"),
Usage: "The maximum duration of time allowed for processing a getFeeStats request. When that time elapses, the rpc server would return -32001 and abort the request's execution",
ConfigKey: &cfg.MaxGetFeeStatsExecutionDuration,
DefaultValue: 5 * time.Second,
},
}
return *cfg.optionsCache
}
Expand Down
32 changes: 17 additions & 15 deletions cmd/soroban-rpc/internal/daemon/daemon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db"
"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events"
"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow"
"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ingest"
"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/preflight"
"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/transactions"
"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/util"
Expand Down Expand Up @@ -103,13 +103,14 @@
// newCaptiveCore creates a new captive core backend instance and returns it.
func newCaptiveCore(cfg *config.Config, logger *supportlog.Entry) (*ledgerbackend.CaptiveStellarCore, error) {
captiveCoreTomlParams := ledgerbackend.CaptiveCoreTomlParams{
HTTPPort: &cfg.CaptiveCoreHTTPPort,
HistoryArchiveURLs: cfg.HistoryArchiveURLs,
NetworkPassphrase: cfg.NetworkPassphrase,
Strict: true,
UseDB: true,
EnforceSorobanDiagnosticEvents: true,
CoreBinaryPath: cfg.StellarCoreBinaryPath,
HTTPPort: &cfg.CaptiveCoreHTTPPort,
HistoryArchiveURLs: cfg.HistoryArchiveURLs,
NetworkPassphrase: cfg.NetworkPassphrase,
Strict: true,
UseDB: true,
EnforceSorobanDiagnosticEvents: true,
EnforceSorobanTransactionMetaExtV1: true,
CoreBinaryPath: cfg.StellarCoreBinaryPath,
}
captiveCoreToml, err := ledgerbackend.NewCaptiveCoreTomlFromFile(cfg.CaptiveCoreConfigPath, captiveCoreTomlParams)
if err != nil {
Expand Down Expand Up @@ -196,12 +197,13 @@
cfg.NetworkPassphrase,
cfg.TransactionLedgerRetentionWindow,
)
feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase)

// initialize the stores using what was on the DB
readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout)
defer cancelReadTxMeta()
// NOTE: We could optimize this to avoid unnecessary ingestion calls
// (the range of txmetads can be larger than the store retention windows)
// (the range of txmetas can be larger than the individual store retention windows)
// but it's probably not worth the pain.
var initialSeq uint32
var currentSeq uint32
Expand All @@ -223,6 +225,9 @@
if err := transactionStore.IngestTransactions(txmeta); err != nil {
logger.WithError(err).Fatal("could not initialize transaction memory store")
}
if err := feewindows.IngestFees(txmeta); err != nil {
logger.WithError(err).Fatal("could not initialize fee stats")
}
return nil
})
if err != nil {
Expand All @@ -237,12 +242,7 @@
onIngestionRetry := func(err error, dur time.Duration) {
logger.WithError(err).Error("could not run ingestion. Retrying")
}
maxRetentionWindow := cfg.EventLedgerRetentionWindow
if cfg.TransactionLedgerRetentionWindow > maxRetentionWindow {
maxRetentionWindow = cfg.TransactionLedgerRetentionWindow
} else if cfg.EventLedgerRetentionWindow == 0 && cfg.TransactionLedgerRetentionWindow > ledgerbucketwindow.DefaultEventLedgerRetentionWindow {
maxRetentionWindow = ledgerbucketwindow.DefaultEventLedgerRetentionWindow
}
maxRetentionWindow := max(cfg.EventLedgerRetentionWindow, cfg.TransactionLedgerRetentionWindow, cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow)

Check failure on line 245 in cmd/soroban-rpc/internal/daemon/daemon.go

View workflow job for this annotation

GitHub Actions / golangci 

undefined: max (typecheck)
ingestService := ingest.NewService(ingest.Config{
Logger: logger,
DB: db.NewReadWriter(dbConn, maxLedgerEntryWriteBatchSize, maxRetentionWindow),
Expand All @@ -254,6 +254,7 @@
Timeout: cfg.IngestionTimeout,
OnIngestionRetry: onIngestionRetry,
Daemon: daemon,
FeeWindows: feewindows,
})

ledgerEntryReader := db.NewLedgerEntryReader(dbConn)
Expand All @@ -271,6 +272,7 @@
Daemon: daemon,
EventStore: eventStore,
TransactionStore: transactionStore,
FeeStatWindows: feewindows,
Logger: logger,
LedgerReader: db.NewLedgerReader(dbConn),
LedgerEntryReader: db.NewLedgerEntryReader(dbConn),
Expand Down
Loading
Loading