From 86ee7dc56f84e5b119b5c33b21d2cce65305d07c Mon Sep 17 00:00:00 2001 From: Vegard Stikbakke Date: Mon, 1 Jul 2024 14:19:25 +0200 Subject: [PATCH] Allow multiple HTTP headers and document some configs --- README.md | 46 +++++++++++++++++++++++++++++++++------------- cmd/main.go | 6 +++--- config/config.go | 14 +++++++------- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index ec408f0..c07e1f8 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,26 @@ -# blockchain node indexer -A program that indexes blockchain data into HTTPS://DUNE.COM by connecting directly to an RPC node +# Blockchain node indexer +A program that indexes blockchain data into http://dune.com by connecting directly to an RPC node. # Limitations - This program works with EVM compatible blockchains, doing direct, EVM-specific JSON-RPC calls to the Node RPC endpoint. # How to use: - There are only 3 required arguments for running the indexer: 1. DUNE_API_KEY: Your Dune API Key, you can get this at: https://dune.com/settings/api 1. BLOCKCHAIN_NAME: The name of the blockchain as configured on Dune (for example: "ethereum" blockchain) 1. RPC_NODE_URL: The URL of the NODE RPC endpoint, for example: https://sepolia.optimism.io/ -For more details see the configuration options section - +For more details see the configuration options section below. -## Docker Container - -You can use our public docker container image and run it as such: +## Docker container +You can run our [public container image on DockerHub](https://hub.docker.com/r/duneanalytics/node-indexer) as such: ```bash docker run -e BLOCKCHAIN_NAME='foo' -e RPC_NODE_URL='http://localhost:8545' -e DUNE_API_KEY='your-key-here' duneanalytics/node-indexer ``` - - ## Binary executable - You can also just build and run a binary executable after cloning this repository: Build the binary for your OS: @@ -37,9 +30,36 @@ $ make build $ BLOCKCHAIN_NAME='foo' RPC_NODE_URL='http://localhost:8545' DUNE_API_KEY='your-key-here' ./indexer ``` -## Configuration Options +Or run it directly with `go run`: +```bash +$ go run cmd/main.go --blockchain-name foo ... +``` +## Configuration options You can see all the configuration options by using the `--help` argument: ```bash docker run duneanalytics/node-indexer --help ``` + +Also, we mention some of the options here: + +### Tuning RPC concurrency +The flag `--rpc-concurrency` (environment variable `RPC_CONCURRENCY`) specifies the number of threads (goroutines) +to run concurrently to perform RPC node requests. Default is 25. + +### RPC poll interval +The flag `--rpc-poll-interval` (environment variable `RPC_POLL_INTERVAL`) specifies the duration to wait before checking +if the RPC node has a new block. Default is `300ms`. + +### Adding extra HTTP headers to RPC requests +If you wish to add HTTP headers to RPC requests you can do so by using the flag `--rpc-http-header` (once per header). + +``` +go run cmd/main.go ... --rpc-http-header header1:value1 --rpc-http-header header2:value2` +``` + +Or with the environment variable `RPC_HTTP_HEADERS='header1:value1|header2:value2|...'`, i.e. a `|` separated list of pairs, +where each pair is separated by `:` (make sure to quote the full string to avoid creating a pipe). +``` +docker run .--env RPC_HTTP_HEADERS='header1:value1|header2:value2' ... duneanalytics/node-indexer: +``` diff --git a/cmd/main.go b/cmd/main.go index 271ba61..3416ebb 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -50,9 +50,9 @@ func main() { var rpcClient jsonrpc.BlockchainClient rpcHTTPHeaders := make(map[string]string) - if cfg.RPCNode.ExtraHTTPHeader != "" { - pair := strings.Split(cfg.RPCNode.ExtraHTTPHeader, ",") - // We've validated this list has two elements + for _, header := range cfg.RPCNode.ExtraHTTPHeaders { + pair := strings.Split(header, ":") + // We've validated this list has two elements in `config.HasError()` key := strings.Trim(pair[0], " ") value := strings.Trim(pair[1], " ") logger.Info("Adding extra HTTP header to RPC requests", "key", key, "value", value) diff --git a/config/config.go b/config/config.go index c67904e..32fe0cb 100644 --- a/config/config.go +++ b/config/config.go @@ -23,19 +23,19 @@ func (d DuneClient) HasError() error { } type RPCClient struct { - NodeURL string `long:"rpc-node-url" env:"RPC_NODE_URL" description:"URL for the blockchain node"` - ExtraHTTPHeader string `long:"rpc-http-header" env:"RPC_HTTP_HEADER" description:"Extra HTTP header to send with RPC requests. On the form 'key,value'"` // nolint:lll - SkipFailedBlocks bool `long:"rpc-skip-failed-blocks" env:"RPC_SKIP_FAILED_BLOCKS" description:"Skip blocks that we fail to get from RPC. If false, we crash on RPC request failure"` // nolint:lll + NodeURL string `long:"rpc-node-url" env:"RPC_NODE_URL" description:"URL for the blockchain node"` + ExtraHTTPHeaders []string `long:"rpc-http-header" env:"RPC_HTTP_HEADERS" env-delim:"|" description:"Extra HTTP headers to send with RPC requests. Each header pair must be on the form 'key:value'"` // nolint:lll + SkipFailedBlocks bool `long:"rpc-skip-failed-blocks" env:"RPC_SKIP_FAILED_BLOCKS" description:"Skip blocks that we fail to get from RPC. If false (default), we crash on RPC request failure"` // nolint:lll } func (r RPCClient) HasError() error { if r.NodeURL == "" { return errors.New("RPC node URL is required") } - if r.ExtraHTTPHeader != "" { - header := strings.Split(r.ExtraHTTPHeader, ",") - if len(header) != 2 { - return fmt.Errorf("invalid rpc http header: expected 'key,value', got '%s'", r.ExtraHTTPHeader) + for _, header := range r.ExtraHTTPHeaders { + pair := strings.Split(header, ":") + if len(pair) != 2 { + return fmt.Errorf("invalid rpc http headers: expected 'key:value', got '%s'", pair) } } return nil