Add DRY_RUN option

duneanalytics · Aug 16, 2024 · 26e4f91 · 26e4f91
1 parent 5431f0f
commit 26e4f91
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 5 deletions.
diff --git a/client/duneapi/client.go b/client/duneapi/client.go
@@ -167,6 +167,9 @@ func (c *client) sendRequest(ctx context.Context, request BlockchainIngestReques
 	}()
 
 	url := fmt.Sprintf("%s/api/beta/blockchain/%s/ingest", c.cfg.URL, c.cfg.BlockchainName)
+	if c.cfg.DryRun {
+		url = fmt.Sprintf("%s/api/beta/blockchain/%s/ingest/dry-run", c.cfg.URL, c.cfg.BlockchainName)
+	}
 	c.log.Debug("Sending request", "url", url)
 	req, err := retryablehttp.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(request.Payload))
 	if err != nil {
@@ -224,6 +227,10 @@ func (c *client) Close() error {
 }
 
 func (c *client) PostProgressReport(ctx context.Context, progress models.BlockchainIndexProgress) error {
+	if c.cfg.DryRun {
+		return nil
+	}
+
 	var request PostBlockchainProgressRequest
 	var err error
 	var responseStatus string
@@ -300,6 +307,15 @@ func (c *client) PostProgressReport(ctx context.Context, progress models.Blockch
 }
 
 func (c *client) GetProgressReport(ctx context.Context) (*models.BlockchainIndexProgress, error) {
+	if c.cfg.DryRun {
+		return &models.BlockchainIndexProgress{
+			BlockchainName:          c.cfg.BlockchainName,
+			EVMStack:                c.cfg.Stack.String(),
+			LastIngestedBlockNumber: -1, // no block ingested
+			LatestBlockNumber:       0,
+		}, nil
+	}
+
 	var response GetBlockchainProgressResponse
 	var err error
 	var responseStatus string
@@ -372,6 +388,10 @@ func (c *client) GetProgressReport(ctx context.Context) (*models.BlockchainIndex
 }
 
 func (c *client) GetBlockGaps(ctx context.Context) (*models.BlockchainGaps, error) {
+	if c.cfg.DryRun {
+		return &models.BlockchainGaps{}, nil
+	}
+
 	var response BlockchainGapsResponse
 	var err error
 	var responseStatus string

diff --git a/client/duneapi/models.go b/client/duneapi/models.go
@@ -23,6 +23,8 @@ type Config struct {
 	// - reduces bandwidth
 	DisableCompression bool
 
+	DryRun bool
+
 	DisableBatchHeader bool // for testing/backwards compatibility
 }
 

diff --git a/cmd/main.go b/cmd/main.go
@@ -58,6 +58,7 @@ func main() {
 		Stack:              cfg.RPCStack,
 		DisableCompression: cfg.DisableCompression,
 		DisableBatchHeader: cfg.Dune.DisableBatchHeader,
+		DryRun:             cfg.DryRun,
 	})
 	if err != nil {
 		stdlog.Fatal(err)
@@ -71,6 +72,7 @@ func main() {
 		BlockchainName:     cfg.BlockchainName,
 		Stack:              cfg.RPCStack,
 		DisableCompression: cfg.DisableCompression,
+		DryRun:             cfg.DryRun,
 	})
 	if err != nil {
 		stdlog.Fatal(err)

diff --git a/config/config.go b/config/config.go
@@ -43,11 +43,12 @@ func (r RPCClient) HasError() error {
 }
 
 type Config struct {
-	BlockHeight            int64  `long:"block-height" env:"BLOCK_HEIGHT" description:"block height to start from" default:"-1"`                         // nolint:lll
-	BlockchainName         string `long:"blockchain-name" env:"BLOCKCHAIN_NAME" description:"name of the blockchain" required:"true"`                    // nolint:lll
-	DisableCompression     bool   `long:"disable-compression" env:"DISABLE_COMPRESSION" description:"disable compression when sending data to Dune"`     // nolint:lll
-	DisableGapsQuery       bool   `long:"disable-gaps-query" env:"DISABLE_GAPS_QUERY" description:"disable gaps query used to populate the initial DLQ"` // nolint:lll
-	DLQOnly                bool   `long:"dlq-only" env:"DLQ_ONLY" description:"Runs just the DLQ processing on its own"`                                 // nolint:lll
+	BlockHeight            int64  `long:"block-height" env:"BLOCK_HEIGHT" description:"block height to start from" default:"-1"`                                   // nolint:lll
+	BlockchainName         string `long:"blockchain-name" env:"BLOCKCHAIN_NAME" description:"name of the blockchain" required:"true"`                              // nolint:lll
+	DisableCompression     bool   `long:"disable-compression" env:"DISABLE_COMPRESSION" description:"disable compression when sending data to Dune"`               // nolint:lll
+	DisableGapsQuery       bool   `long:"disable-gaps-query" env:"DISABLE_GAPS_QUERY" description:"disable gaps query used to populate the initial DLQ"`           // nolint:lll
+	DLQOnly                bool   `long:"dlq-only" env:"DLQ_ONLY" description:"Runs just the DLQ processing on its own"`                                           // nolint:lll
+	DryRun                 bool   `long:"dry-run" env:"DRY_RUN" description:"When enabled, data is sent to Dune for validation but is not written to Dune tables"` // nolint:lll
 	Dune                   DuneClient
 	PollInterval           time.Duration `long:"rpc-poll-interval" env:"RPC_POLL_INTERVAL" description:"Interval to poll the blockchain node" default:"300ms"`    // nolint:lll
 	PollDLQInterval        time.Duration `long:"dlq-poll-interval" env:"DLQ_POLL_INTERVAL" description:"Interval to poll the dlq" default:"300ms"`                // nolint:lll