Test main loop #6
+452
−27
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,9 +8,11 @@ TEST_TIMEOUT := 10s | |
|
|
||
| all: lint test build | ||
|
|
||
| setup: bin/golangci-lint bin/gofumpt bin/moq | ||
| go mod download | ||
|
|
||
| bin/moq: | ||
| GOBIN=$(PWD)/bin go install github.com/matryer/[email protected] | ||
| bin/golangci-lint: | ||
| GOBIN=$(PWD)/bin go install github.com/golangci/golangci-lint/cmd/[email protected] | ||
| bin/gofumpt: bin | ||
|
|
@@ -30,6 +32,11 @@ test: | |
| go mod tidy | ||
| go test -timeout=$(TEST_TIMEOUT) -race -bench=. -benchmem -cover ./... | ||
|
|
||
| gen-mocks: bin/moq ./client/jsonrpc/ ./client/duneapi/ | ||
| ./bin/moq -pkg jsonrpc_mock -out ./mocks/jsonrpc/rpcnode.go ./client/jsonrpc BlockchainClient | ||
| ./bin/moq -pkg duneapi_mock -out ./mocks/duneapi/client.go ./client/duneapi BlockchainIngester | ||
|
|
||
|
|
||
| image-build: | ||
| @echo "# Building ingester docker image..." | ||
| docker build -t $(APPLICATION) -f Dockerfile --build-arg GITHUB_TOKEN=${GITHUB_TOKEN} . | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,15 +1,180 @@ | ||
| package ingester_test | ||
|
|
||
| import ( | ||
| "context" | ||
| "io" | ||
| "log/slog" | ||
| "sync/atomic" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/duneanalytics/blockchain-ingester/ingester" | ||
| duneapi_mock "github.com/duneanalytics/blockchain-ingester/mocks/duneapi" | ||
| jsonrpc_mock "github.com/duneanalytics/blockchain-ingester/mocks/jsonrpc" | ||
| "github.com/duneanalytics/blockchain-ingester/models" | ||
| "github.com/go-errors/errors" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| func TestBlockConsumptionLoopErrors(t *testing.T) { | ||
|
There was a problem hiding this comment. It's not immediately clear to me what this test is testing. Is it testing that we exit the function cleanly under these scenarios + what blocks have been ingested? Or am I misunderstanding? There was a problem hiding this comment. this test is bogus right now, because I realized (while trying to get this test any decent) that the logic I created is just bad... The problem is:
in short, this test just shows we're a bit confused on what to expect from the test. |
||
| testcases := []struct { | ||
| name string | ||
| LatestIsBroken bool | ||
| BlockByNumberIsBroken bool | ||
| }{ | ||
| { | ||
| name: "we're up to date, following the head", | ||
| LatestIsBroken: false, | ||
| BlockByNumberIsBroken: false, | ||
| }, | ||
| { | ||
| name: "the RPC node is broken, all API calls are failing", | ||
| LatestIsBroken: true, | ||
| BlockByNumberIsBroken: true, | ||
| }, | ||
| { | ||
| name: "BlockByNumber, a specific jsonRPC on the RPC node is broken", | ||
| LatestIsBroken: false, | ||
| BlockByNumberIsBroken: true, | ||
| }, | ||
| } | ||
|
|
||
| for _, tc := range testcases { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| if tc.LatestIsBroken { | ||
| t.Skip("latest block number is broken, we don't behave correctly yet") | ||
| } | ||
| ctx, cancel := context.WithCancel(context.Background()) | ||
| maxBlockNumber := int64(100) | ||
| producedBlockNumber := int64(0) | ||
| rpcClient := &jsonrpc_mock.BlockchainClientMock{ | ||
| LatestBlockNumberFunc: func() (int64, error) { | ||
| if tc.LatestIsBroken { | ||
| return 0, errors.New("latest block number is broken") | ||
| } | ||
| return maxBlockNumber, nil | ||
| }, | ||
| BlockByNumberFunc: func(_ context.Context, blockNumber int64) (models.RPCBlock, error) { | ||
| if tc.BlockByNumberIsBroken { | ||
| return models.RPCBlock{}, errors.New("block by number is broken") | ||
| } | ||
| if blockNumber > maxBlockNumber { | ||
| // end tests | ||
| cancel() | ||
| } | ||
| atomic.StoreInt64(&producedBlockNumber, blockNumber) | ||
| return models.RPCBlock{ | ||
| BlockNumber: blockNumber, | ||
| Payload: []byte(`block`), | ||
| }, nil | ||
| }, | ||
| } | ||
| ing := ingester.New(slog.New(slog.NewTextHandler(io.Discard, nil)), rpcClient, nil, ingester.Config{ | ||
| MaxBatchSize: 1, | ||
| PollInterval: 1000 * time.Millisecond, | ||
| }) | ||
|
|
||
| outCh := make(chan models.RPCBlock, maxBlockNumber+1) | ||
| defer close(outCh) | ||
| err := ing.ConsumeBlocks(ctx, outCh, 0, maxBlockNumber) | ||
| require.Error(t, err) // this is expected | ||
| if tc.BlockByNumberIsBroken { | ||
| require.Equal(t, producedBlockNumber, int64(0)) | ||
| } | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestBlockSendingLoop(t *testing.T) { | ||
| testcases := []string{ | ||
| "we're up to date, following the head", | ||
| "we're failing intermittently, the Dune API is broken", | ||
| "we're erroring systematically, the Dune API is down", | ||
| } | ||
| for _, testcase := range testcases { | ||
| t.Run(testcase, func(t *testing.T) { | ||
| t.Skip("not implemented") | ||
|
|
||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestRunLoopBaseCase(t *testing.T) { | ||
|
|
||
| testCases := []struct { | ||
| name string | ||
| i int64 | ||
| }{ | ||
| {name: "1 block", i: 1}, | ||
| {name: "100 blocks", i: 100}, | ||
| } | ||
| sentBlockNumber := int64(0) | ||
| producedBlockNumber := int64(0) | ||
| duneapi := &duneapi_mock.BlockchainIngesterMock{ | ||
| SendBlockFunc: func(block models.RPCBlock) error { | ||
| atomic.StoreInt64(&sentBlockNumber, block.BlockNumber) | ||
| return nil | ||
| }, | ||
| } | ||
| rpcClient := &jsonrpc_mock.BlockchainClientMock{ | ||
| LatestBlockNumberFunc: func() (int64, error) { | ||
| return 1000, nil | ||
| }, | ||
| BlockByNumberFunc: func(_ context.Context, blockNumber int64) (models.RPCBlock, error) { | ||
| atomic.StoreInt64(&producedBlockNumber, blockNumber) | ||
| return models.RPCBlock{ | ||
| BlockNumber: blockNumber, | ||
| Payload: []byte(`block`), | ||
| }, nil | ||
| }, | ||
| } | ||
| for _, tc := range testCases { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| ing := ingester.New(slog.New(slog.NewTextHandler(io.Discard, nil)), rpcClient, duneapi, ingester.Config{ | ||
| MaxBatchSize: 1, | ||
| PollInterval: 1000 * time.Millisecond, | ||
| }) | ||
|
|
||
| err := ing.Run(context.Background(), 0, tc.i) | ||
| require.NoError(t, err) | ||
| require.Equal(t, producedBlockNumber, tc.i) | ||
| require.Equal(t, sentBlockNumber, tc.i) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestRunLoopUntilCancel(t *testing.T) { | ||
| ctx, cancel := context.WithCancel(context.Background()) | ||
| maxBlockNumber := int64(1000) | ||
| sentBlockNumber := int64(0) | ||
| producedBlockNumber := int64(0) | ||
| duneapi := &duneapi_mock.BlockchainIngesterMock{ | ||
| SendBlockFunc: func(block models.RPCBlock) error { | ||
| atomic.StoreInt64(&sentBlockNumber, block.BlockNumber) | ||
| if block.BlockNumber == maxBlockNumber { | ||
| // cancel execution when we send the last block | ||
| cancel() | ||
| } | ||
| return nil | ||
| }, | ||
| } | ||
| rpcClient := &jsonrpc_mock.BlockchainClientMock{ | ||
| LatestBlockNumberFunc: func() (int64, error) { | ||
| return maxBlockNumber + 1, nil | ||
| }, | ||
| BlockByNumberFunc: func(_ context.Context, blockNumber int64) (models.RPCBlock, error) { | ||
| atomic.StoreInt64(&producedBlockNumber, blockNumber) | ||
| return models.RPCBlock{ | ||
| BlockNumber: blockNumber, | ||
| Payload: []byte(`block`), | ||
| }, nil | ||
| }, | ||
| } | ||
| ing := ingester.New(slog.New(slog.NewTextHandler(io.Discard, nil)), rpcClient, duneapi, ingester.Config{ | ||
| MaxBatchSize: 1, | ||
| PollInterval: 1000 * time.Millisecond, | ||
| }) | ||
|
|
||
| err := ing.Run(ctx, 0, maxBlockNumber) | ||
| require.ErrorIs(t, err, context.Canceled) | ||
| require.Equal(t, producedBlockNumber, maxBlockNumber) | ||
| require.Equal(t, sentBlockNumber, maxBlockNumber) | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I didn't realize
/* */was valid in Go!