Allow multiple HTTP headers and document some configs

README.md

@@ -1,33 +1,26 @@
-# blockchain node indexer
-A program that indexes blockchain data into HTTPS://DUNE.COM by connecting directly to an RPC node
+# Blockchain node indexer
+A program that indexes blockchain data into http://dune.com by connecting directly to an RPC node.
 
 # Limitations
-
 This program works with EVM compatible blockchains, doing direct, EVM-specific JSON-RPC calls to the Node RPC endpoint.
 
 
 # How to use:
-
 There are only 3 required arguments for running the indexer:
   1. DUNE_API_KEY: Your Dune API Key, you can get this at: https://dune.com/settings/api
   1. BLOCKCHAIN_NAME: The name of the blockchain as configured on Dune (for example: "ethereum" blockchain)
   1. RPC_NODE_URL: The URL of the NODE RPC endpoint, for example: https://sepolia.optimism.io/
 
-For more details see the configuration options section
-
+For more details see the configuration options section below.
 
-## Docker Container
-
-You can use our public docker container image and run it as such:
+## Docker container
+You can run our [public container image on DockerHub](https://hub.docker.com/r/duneanalytics/node-indexer) as such:
 
 ```bash
 docker run -e BLOCKCHAIN_NAME='foo' -e RPC_NODE_URL='http://localhost:8545' -e DUNE_API_KEY='your-key-here' duneanalytics/node-indexer
 ```
 
-
-
 ## Binary executable
-
 You can also just build and run a binary executable after cloning this repository:
 
 Build the binary for your OS:
@@ -37,9 +30,36 @@ $ make build
 $ BLOCKCHAIN_NAME='foo' RPC_NODE_URL='http://localhost:8545' DUNE_API_KEY='your-key-here' ./indexer
 ```
 
-## Configuration Options
+Or run it directly with `go run`:
+```bash
+$ go run cmd/main.go --blockchain-name foo ...
+```
 
+## Configuration options
 You can see all the configuration options by using the `--help` argument:
 ```bash
 docker run duneanalytics/node-indexer --help
 ```
+
+Also, we mention some of the options here:
+
+### Tuning RPC concurrency
+The flag `--rpc-concurrency` (environment variable `RPC_CONCURRENCY`) specifies the number of threads (goroutines)
+to run concurrently to perform RPC node requests. Default is 25.
+
+### RPC poll interval
+The flag `--rpc-poll-interval` (environment variable `RPC_POLL_INTERVAL`) specifies the duration to wait before checking
+if the RPC node has a new block. Default is `300ms`.
+
+### Adding extra HTTP headers to RPC requests
+If you wish to add HTTP headers to RPC requests you can do so by using the flag `--rpc-http-header` (once per header).
+
+```
+go run cmd/main.go ... --rpc-http-header header1:value1 --rpc-http-header header2:value2`
+```
+
+Or with the environment variable `RPC_HTTP_HEADERS='header1:value1|header2:value2|...'`, i.e. a `|` separated list of pairs,
+where each pair is separated by `:` (make sure to quote the full string to avoid creating a pipe).
+```
+docker run .--env RPC_HTTP_HEADERS='header1:value1|header2:value2' ... duneanalytics/node-indexer:<version>
+```

cmd/main.go

@@ -50,9 +50,9 @@ func main() {
 	var rpcClient jsonrpc.BlockchainClient
 
 	rpcHTTPHeaders := make(map[string]string)
-	if cfg.RPCNode.ExtraHTTPHeader != "" {
-		pair := strings.Split(cfg.RPCNode.ExtraHTTPHeader, ",")
-		// We've validated this list has two elements
+	for _, header := range cfg.RPCNode.ExtraHTTPHeaders {
+		pair := strings.Split(header, ":")
+		// We've validated this list has two elements in `config.HasError()`
 		key := strings.Trim(pair[0], " ")
 		value := strings.Trim(pair[1], " ")
 		logger.Info("Adding extra HTTP header to RPC requests", "key", key, "value", value)

config/config.go

@@ -23,19 +23,19 @@ func (d DuneClient) HasError() error {
 }
 
 type RPCClient struct {
-	NodeURL          string `long:"rpc-node-url" env:"RPC_NODE_URL" description:"URL for the blockchain node"`
-	ExtraHTTPHeader  string `long:"rpc-http-header" env:"RPC_HTTP_HEADER" description:"Extra HTTP header to send with RPC requests. On the form 'key,value'"`                              // nolint:lll
-	SkipFailedBlocks bool   `long:"rpc-skip-failed-blocks" env:"RPC_SKIP_FAILED_BLOCKS" description:"Skip blocks that we fail to get from RPC. If false, we crash on RPC request failure"` // nolint:lll
+	NodeURL          string   `long:"rpc-node-url" env:"RPC_NODE_URL" description:"URL for the blockchain node"`
+	ExtraHTTPHeaders []string `long:"rpc-http-header" env:"RPC_HTTP_HEADERS" env-delim:"|" description:"Extra HTTP headers to send with RPC requests. Each header pair must be on the form 'key:value'"` // nolint:lll
+	SkipFailedBlocks bool     `long:"rpc-skip-failed-blocks" env:"RPC_SKIP_FAILED_BLOCKS" description:"Skip blocks that we fail to get from RPC. If false (default), we crash on RPC request failure"`   // nolint:lll
 }
 
 func (r RPCClient) HasError() error {
 	if r.NodeURL == "" {
 		return errors.New("RPC node URL is required")
 	}
-	if r.ExtraHTTPHeader != "" {
-		header := strings.Split(r.ExtraHTTPHeader, ",")
-		if len(header) != 2 {
-			return fmt.Errorf("invalid rpc http header: expected 'key,value', got '%s'", r.ExtraHTTPHeader)
+	for _, header := range r.ExtraHTTPHeaders {
+		pair := strings.Split(header, ":")
+		if len(pair) != 2 {
+			return fmt.Errorf("invalid rpc http headers: expected 'key:value', got '%s'", pair)
 		}
 	}
 	return nil