Merge branch 'develop' into hotfix/2.15.2

.circleci/config.yml

@@ -46,11 +46,12 @@ jobs:
       go_version:
         type: string
     environment:
-      CI_E2E_FILENAME: "faab6dcf/rel-nightly"
+      CI_E2E_FILENAME: "fa6ad40d/rel-nightly"
     steps:
       - go/install:
           version: << parameters.go_version >>
       - install_dependencies
+      - install_linter
       - run_tests
       - codecov/upload
   test_nightly:
@@ -65,6 +66,7 @@ jobs:
       - go/install:
           version: << parameters.go_version >>
       - install_dependencies
+      - install_linter
       - run_tests
       - codecov/upload
       - slack/notify: &slack-fail-event
@@ -114,15 +116,19 @@ commands:
             sudo apt update
             sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-math-dev libffi-dev
             pip3 install -r misc/requirements.txt
+            pip3 install e2e_tests/
 
       - run:
           name: sync submodules (go-algorand)
           command: |
             git submodule sync
             git submodule update --init
 
-      - run: echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV
+      - run: echo 'export PATH=$PATH:/usr/local/go/bin:$HOME/.local/bin' >> $BASH_ENV
 
+  install_linter:
+    description: Install golangci-lint
+    steps:
       - run:
           name: Install golangci-lint
           command: go install github.com/golangci/golangci-lint/cmd/[email protected]
@@ -143,6 +149,7 @@ commands:
       - run: make test-generate
       - run: make fakepackage
       - run: make e2e
+      - run: make e2e-conduit
 
   run_indexer_vs_algod:
     steps:

.codecov.yml

@@ -2,6 +2,11 @@ codecov:
   require_ci_to_pass: no
   branch: develop
 
+ignore:
+  - "idb/mocks"
+  - "idb/dummy"
+  - "util/test"
+
 coverage:
   precision: 2
   round: down

.gitignore

@@ -1,8 +1,10 @@
 # Build artifacts
 cmd/algorand-indexer/algorand-indexer
+cmd/conduit/conduit
 api/.3tmp.json
 api/generate
 tmp/
+./conduit-docs/
 
 # Binaries for programs and plugins
 *.exe
@@ -28,6 +30,9 @@ _*.json
 
 # Python
 __pycache__
+e2e_tests/dist
+e2e_tests/build
+e2e_tests/*egg-info*
 .venv
 
 # jetbrains IDE
@@ -48,3 +53,6 @@ coverage.txt
 
 # asdf
 .tool-versions
+
+# conduit example
+cmd/conduit/data

Makefile

@@ -22,7 +22,15 @@ COVERPKG := $(shell go list ./...  | grep -v '/cmd/' | egrep -v '(testing|test|m
 # Used for e2e test
 export GO_IMAGE = golang:$(shell go version | cut -d ' ' -f 3 | tail -c +3 )
 
-# This is the default target, build the indexer:
+# This is the default target, build everything:
+all: conduit cmd/algorand-indexer/algorand-indexer go-algorand idb/postgres/internal/schema/setup_postgres_sql.go idb/mocks/IndexerDb.go
+
+conduit: go-algorand conduit-docs
+	go generate ./... && cd cmd/conduit && go build -ldflags="${GOLDFLAGS}"
+
+conduit-docs:
+	go install ./cmd/conduit-docs/
+
 cmd/algorand-indexer/algorand-indexer: idb/postgres/internal/schema/setup_postgres_sql.go go-algorand
 	cd cmd/algorand-indexer && go build -ldflags="${GOLDFLAGS}"
 
@@ -47,7 +55,7 @@ package: go-algorand
 	misc/release.py --host-only --outdir $(PKG_DIR)
 
 # used in travis test builds; doesn't verify that tag and .version match
-fakepackage: go-algorand
+fakepackage: go-algorand conduit-docs
 	rm -rf $(PKG_DIR)
 	mkdir -p $(PKG_DIR)
 	misc/release.py --host-only --outdir $(PKG_DIR) --fake-release
@@ -67,8 +75,18 @@ integration: cmd/algorand-indexer/algorand-indexer
 	curl -s https://algorand-testdata.s3.amazonaws.com/indexer/test_blockdata/create_destroy.tar.bz2 -o test/blockdata/create_destroy.tar.bz2
 	test/postgres_integration_test.sh
 
-e2e: cmd/algorand-indexer/algorand-indexer
-	cd misc && docker-compose build --build-arg GO_IMAGE=${GO_IMAGE} && docker-compose up --exit-code-from e2e
+# note: when running e2e tests manually be sure to set the e2e filename:
+# 	'export CI_E2E_FILENAME=rel-nightly'
+# To keep the container running at exit set 'export EXTRA="--keep-alive"',
+# once the container is paused use 'docker exec <id> bash' to inspect temp
+# files in `/tmp/*/'
+e2e: cmd/algorand-indexer/algorand-indexer conduit-docs
+	cd e2e_tests/docker/indexer/ && docker-compose build --build-arg GO_IMAGE=${GO_IMAGE} && docker-compose up --exit-code-from e2e
+
+# note: when running e2e tests manually be sure to set the e2e filename: 'export CI_E2E_FILENAME=rel-nightly'
+e2e-conduit: conduit
+	cd third_party/go-algorand && make install
+	export PATH=$(PATH):$(shell go env GOPATH)/bin; pip3 install e2e_tests/ && e2econduit --s3-source-net ${CI_E2E_FILENAME} --conduit-bin cmd/conduit/conduit
 
 deploy:
 	mule/deploy.sh
@@ -96,9 +114,10 @@ indexer-v-algod: nightly-setup indexer-v-algod-swagger nightly-teardown
 # fetch and update submodule. it's default to latest rel/nightly branch.
 # to use a different branch, update the branch in .gitmodules for CI build,
 # and for local testing, you may checkout a specific branch in the submodule.
-# after submodule is updated, CI_E2E_FILE in circleci/config.yml should also
-# be updated to use a newer artifact. path copied from s3 bucket, s3://algorand-testdata/indexer/e2e4/
+# after submodule is updated, CI_E2E_FILENAME in .circleci/config.yml should
+# also be updated to use a newer artifact. path copied from s3 bucket,
+# s3://algorand-testdata/indexer/e2e4/
 update-submodule:
 	git submodule update --remote
 
-.PHONY: test e2e integration fmt lint deploy sign test-package package fakepackage cmd/algorand-indexer/algorand-indexer idb/mocks/IndexerDb.go go-algorand indexer-v-algod
+.PHONY: all test e2e integration fmt lint deploy sign test-package package fakepackage cmd/algorand-indexer/algorand-indexer idb/mocks/IndexerDb.go go-algorand indexer-v-algod conduit conduit-docs

README.md

@@ -48,11 +48,11 @@ Indexer is part of the [sandbox](https://github.com/algorand/sandbox) private ne
 - Search and filter accounts, transactions, assets, and asset balances with many different parameters.
 - Pagination of results.
 - Enriched transaction and account data:
-    - Confirmation round (block containing the transaction)
-    - Confirmation time
-    - Signature type
-    - Close amounts
-    - Create/delete rounds.
+  - Confirmation round (block containing the transaction)
+  - Confirmation time
+  - Signature type
+  - Close amounts
+  - Create/delete rounds.
 - Human readable field names instead of the space optimized protocol level names.
 
 # Contributing
@@ -74,6 +74,9 @@ There are two primary modes of operation:
 
 In both configurations, a postgres connection string is required. Both DSN and URL formats are supported, [details are available here](https://pkg.go.dev/github.com/jackc/pgx/v4/[email protected]#ParseConfig).
 
+In addition, the indexer uses a data directory that stores data needed for runtime operation and configuration.
+See the [Data Directory documentation](docs/DataDirectory.md) for how to (re-)initialize this directory in case it is lost or needs to be re-created.
+
 ### Database updater
 In this mode, the database will be populated with data fetched from an [Algorand archival node](https://developer.algorand.org/docs/run-a-node/setup/types/#archival-mode). Because every block must be fetched to bootstrap the database, the initial import for a ledger with a long history will take a while. If the daemon is terminated, it will resume processing wherever it left off.
 

accounting/accounting.go

@@ -1,26 +1,24 @@
 package accounting
 
 import (
-	"github.com/algorand/go-algorand/data/basics"
-	"github.com/algorand/go-algorand/data/transactions"
-	"github.com/algorand/go-algorand/protocol"
+	sdk "github.com/algorand/go-algorand-sdk/v2/types"
 )
 
 // GetTransactionParticipants calls function `add` for every address referenced in the
 // given transaction, possibly with repetition.
-func GetTransactionParticipants(stxnad *transactions.SignedTxnWithAD, includeInner bool, add func(address basics.Address)) {
+func GetTransactionParticipants(stxnad *sdk.SignedTxnWithAD, includeInner bool, add func(address sdk.Address)) {
 	txn := &stxnad.Txn
 
 	add(txn.Sender)
 
 	switch txn.Type {
-	case protocol.PaymentTx:
+	case sdk.PaymentTx:
 		add(txn.Receiver)
 		// Close address is optional.
 		if !txn.CloseRemainderTo.IsZero() {
 			add(txn.CloseRemainderTo)
 		}
-	case protocol.AssetTransferTx:
+	case sdk.AssetTransferTx:
 		// If asset sender is non-zero, it is a clawback transaction. Otherwise,
 		// the transaction sender address is used.
 		if !txn.AssetSender.IsZero() {
@@ -31,9 +29,9 @@ func GetTransactionParticipants(stxnad *transactions.SignedTxnWithAD, includeInn
 		if !txn.AssetCloseTo.IsZero() {
 			add(txn.AssetCloseTo)
 		}
-	case protocol.AssetFreezeTx:
+	case sdk.AssetFreezeTx:
 		add(txn.FreezeAccount)
-	case protocol.ApplicationCallTx:
+	case sdk.ApplicationCallTx:
 		for _, address := range txn.ApplicationCallTxnFields.Accounts {
 			add(address)
 		}

accounting/rewind.go

@@ -4,12 +4,11 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/algorand/go-algorand/data/basics"
-	"github.com/algorand/go-algorand/data/transactions"
-	"github.com/algorand/go-algorand/protocol"
 	models "github.com/algorand/indexer/api/generated/v2"
-
 	"github.com/algorand/indexer/idb"
+	"github.com/algorand/indexer/types"
+
+	sdk "github.com/algorand/go-algorand-sdk/v2/types"
 )
 
 // ConsistencyError is returned when the database returns inconsistent (stale) results.
@@ -60,15 +59,15 @@ func (sare *SpecialAccountRewindError) Error() string {
 	return fmt.Sprintf("unable to rewind the %s", sare.account)
 }
 
-var specialAccounts *transactions.SpecialAddresses
+var specialAccounts *types.SpecialAddresses
 
 // AccountAtRound queries the idb.IndexerDb object for transactions and rewinds most fields of the account back to
 // their values at the requested round.
 // `round` must be <= `account.Round`
 func AccountAtRound(ctx context.Context, account models.Account, round uint64, db idb.IndexerDb) (acct models.Account, err error) {
 	// Make sure special accounts cache has been initialized.
 	if specialAccounts == nil {
-		var accounts transactions.SpecialAddresses
+		var accounts types.SpecialAddresses
 		accounts, err = db.GetSpecialAccounts(ctx)
 		if err != nil {
 			return models.Account{}, fmt.Errorf("unable to get special accounts: %v", err)
@@ -77,8 +76,8 @@ func AccountAtRound(ctx context.Context, account models.Account, round uint64, d
 	}
 
 	acct = account
-	var addr basics.Address
-	addr, err = basics.UnmarshalChecksumAddress(account.Address)
+	var addr sdk.Address
+	addr, err = sdk.DecodeAddress(account.Address)
 	if err != nil {
 		return
 	}
@@ -126,34 +125,34 @@ func AccountAtRound(ctx context.Context, account models.Account, round uint64, d
 				fmt.Errorf("rewinding past inner transactions is not supported")
 		}
 		if addr == stxn.Txn.Sender {
-			acct.AmountWithoutPendingRewards += stxn.Txn.Fee.ToUint64()
-			acct.AmountWithoutPendingRewards -= stxn.SenderRewards.ToUint64()
+			acct.AmountWithoutPendingRewards += uint64(stxn.Txn.Fee)
+			acct.AmountWithoutPendingRewards -= uint64(stxn.SenderRewards)
 		}
 		switch stxn.Txn.Type {
-		case protocol.PaymentTx:
+		case sdk.PaymentTx:
 			if addr == stxn.Txn.Sender {
-				acct.AmountWithoutPendingRewards += stxn.Txn.Amount.ToUint64()
+				acct.AmountWithoutPendingRewards += uint64(stxn.Txn.Amount)
 			}
 			if addr == stxn.Txn.Receiver {
-				acct.AmountWithoutPendingRewards -= stxn.Txn.Amount.ToUint64()
-				acct.AmountWithoutPendingRewards -= stxn.ReceiverRewards.ToUint64()
+				acct.AmountWithoutPendingRewards -= uint64(stxn.Txn.Amount)
+				acct.AmountWithoutPendingRewards -= uint64(stxn.ReceiverRewards)
 			}
 			if addr == stxn.Txn.CloseRemainderTo {
 				// unwind receiving a close-to
-				acct.AmountWithoutPendingRewards -= stxn.ClosingAmount.ToUint64()
-				acct.AmountWithoutPendingRewards -= stxn.CloseRewards.ToUint64()
+				acct.AmountWithoutPendingRewards -= uint64(stxn.ClosingAmount)
+				acct.AmountWithoutPendingRewards -= uint64(stxn.CloseRewards)
 			} else if !stxn.Txn.CloseRemainderTo.IsZero() {
 				// unwind sending a close-to
-				acct.AmountWithoutPendingRewards += stxn.ClosingAmount.ToUint64()
+				acct.AmountWithoutPendingRewards += uint64(stxn.ClosingAmount)
 			}
-		case protocol.KeyRegistrationTx:
+		case sdk.KeyRegistrationTx:
 			// TODO: keyreg does not rewind. workaround: query for txns on an account with typeenum=2 to find previous values it was set to.
-		case protocol.AssetConfigTx:
+		case sdk.AssetConfigTx:
 			if stxn.Txn.ConfigAsset == 0 {
 				// create asset, unwind the application of the value
 				assetUpdate(&acct, txnrow.AssetID, 0, stxn.Txn.AssetParams.Total)
 			}
-		case protocol.AssetTransferTx:
+		case sdk.AssetTransferTx:
 			if addr == stxn.Txn.AssetSender || addr == stxn.Txn.Sender {
 				assetUpdate(&acct, uint64(stxn.Txn.XferAsset), stxn.Txn.AssetAmount+txnrow.Extra.AssetCloseAmount, 0)
 			}
@@ -163,7 +162,7 @@ func AccountAtRound(ctx context.Context, account models.Account, round uint64, d
 			if addr == stxn.Txn.AssetCloseTo {
 				assetUpdate(&acct, uint64(stxn.Txn.XferAsset), 0, txnrow.Extra.AssetCloseAmount)
 			}
-		case protocol.AssetFreezeTx:
+		case sdk.AssetFreezeTx:
 		default:
 			err = fmt.Errorf("%s[%d,%d]: rewinding past txn type %s is not currently supported", account.Address, txnrow.Round, txnrow.Intra, stxn.Txn.Type)
 			return

accounting/rewind_test.go

@@ -5,9 +5,10 @@ import (
 	"errors"
 	"testing"
 
+	sdk "github.com/algorand/go-algorand-sdk/v2/types"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/transactions"
-	"github.com/algorand/go-algorand/protocol"
+	"github.com/algorand/indexer/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 
@@ -17,7 +18,7 @@ import (
 )
 
 func TestBasic(t *testing.T) {
-	var a basics.Address
+	var a sdk.Address
 	a[0] = 'a'
 
 	account := models.Account{
@@ -29,13 +30,13 @@ func TestBasic(t *testing.T) {
 
 	txnRow := idb.TxnRow{
 		Round: 7,
-		Txn: &transactions.SignedTxnWithAD{
-			SignedTxn: transactions.SignedTxn{
-				Txn: transactions.Transaction{
-					Type: protocol.PaymentTx,
-					PaymentTxnFields: transactions.PaymentTxnFields{
+		Txn: &sdk.SignedTxnWithAD{
+			SignedTxn: sdk.SignedTxn{
+				Txn: sdk.Transaction{
+					Type: sdk.PaymentTx,
+					PaymentTxnFields: sdk.PaymentTxnFields{
 						Receiver: a,
-						Amount:   basics.MicroAlgos{Raw: 2},
+						Amount:   sdk.MicroAlgos(2),
 					},
 				},
 			},
@@ -48,7 +49,7 @@ func TestBasic(t *testing.T) {
 	var outCh <-chan idb.TxnRow = ch
 
 	db := &mocks.IndexerDb{}
-	db.On("GetSpecialAccounts", mock.Anything).Return(transactions.SpecialAddresses{}, nil)
+	db.On("GetSpecialAccounts", mock.Anything).Return(types.SpecialAddresses{}, nil)
 	db.On("Transactions", mock.Anything, mock.Anything).Return(outCh, uint64(8))
 
 	account, err := AccountAtRound(context.Background(), account, 6, db)