AVM: Add support for Boxes

.golangci.yml

@@ -33,6 +33,8 @@ linters-settings:
     - (*github.com/algorand/go-algorand/data/transactions/logic.OpStream).error
     - (*github.com/algorand/go-algorand/data/transactions/logic.OpStream).warnf
     - (*github.com/algorand/go-algorand/data/transactions/logic.OpStream).warn
+    # We do this 121 times and never check the error.
+    - (*github.com/spf13/cobra.Command).MarkFlagRequired
   govet:
     settings:
       printf:

catchup/ledgerFetcher.go

@@ -146,10 +146,12 @@ func (lf *ledgerFetcher) getPeerLedger(ctx context.Context, peer network.HTTPPee
 			"writing balances to disk took %d seconds, "+
 				"writing creatables to disk took %d seconds, "+
 				"writing hashes to disk took %d seconds, "+
+				"writing kv pairs to disk took %d seconds, "+
 				"total duration is %d seconds",
 			downloadProgress.BalancesWriteDuration/time.Second,
 			downloadProgress.CreatablesWriteDuration/time.Second,
 			downloadProgress.HashesWriteDuration/time.Second,
+			downloadProgress.KVWriteDuration/time.Second,
 			writeDuration/time.Second)
 	}
 
@@ -191,5 +193,5 @@ func (lf *ledgerFetcher) getPeerLedger(ctx context.Context, peer network.HTTPPee
 }
 
 func (lf *ledgerFetcher) processBalancesBlock(ctx context.Context, sectionName string, bytes []byte, downloadProgress *ledger.CatchpointCatchupAccessorProgress) error {
-	return lf.accessor.ProgressStagingBalances(ctx, sectionName, bytes, downloadProgress)
+	return lf.accessor.ProcessStagingBalances(ctx, sectionName, bytes, downloadProgress)
 }

cmd/catchpointdump/file.go

@@ -21,6 +21,7 @@ import (
 	"bufio"
 	"context"
 	"database/sql"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -34,6 +35,7 @@ import (
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/bookkeeping"
+	"github.com/algorand/go-algorand/data/transactions/logic"
 	"github.com/algorand/go-algorand/ledger"
 	"github.com/algorand/go-algorand/ledger/ledgercore"
 	"github.com/algorand/go-algorand/logging"
@@ -128,6 +130,10 @@ var fileCmd = &cobra.Command{
 			if err != nil {
 				reportErrorf("Unable to print account database : %v", err)
 			}
+			err = printKeyValueStore("./ledger.tracker.sqlite", outFile)
+			if err != nil {
+				reportErrorf("Unable to print key value store : %v", err)
+			}
 		}
 	},
 }
@@ -176,7 +182,7 @@ func loadCatchpointIntoDatabase(ctx context.Context, catchupAccessor ledger.Catc
 				return fileHeader, err
 			}
 		}
-		err = catchupAccessor.ProgressStagingBalances(ctx, header.Name, balancesBlockBytes, &downloadProgress)
+		err = catchupAccessor.ProcessStagingBalances(ctx, header.Name, balancesBlockBytes, &downloadProgress)
 		if err != nil {
 			return fileHeader, err
 		}
@@ -380,7 +386,65 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
 		}
 
 		// increase the deadline warning to disable the warning message.
-		db.ResetTransactionWarnDeadline(ctx, tx, time.Now().Add(5*time.Second))
+		_, _ = db.ResetTransactionWarnDeadline(ctx, tx, time.Now().Add(5*time.Second))
 		return err
 	})
 }
+
+func printKeyValue(writer *bufio.Writer, key, value []byte) {
+	var pretty string
+	ai, rest, err := logic.SplitBoxKey(string(key))
+	if err == nil {
+		pretty = fmt.Sprintf("box(%d, %s)", ai, base64.StdEncoding.EncodeToString([]byte(rest)))
+	} else {
+		pretty = base64.StdEncoding.EncodeToString(key)
+	}
+
+	fmt.Fprintf(writer, "%s : %v\n", pretty, base64.StdEncoding.EncodeToString(value))
+}
+
+func printKeyValueStore(databaseName string, outFile *os.File) error {
+	fmt.Printf("\n")
+	printDumpingCatchpointProgressLine(0, 50, 0)
+	lastProgressUpdate := time.Now()
+	progress := uint64(0)
+	defer printDumpingCatchpointProgressLine(0, 0, 0)
+
+	fileWriter := bufio.NewWriterSize(outFile, 1024*1024)
+	defer fileWriter.Flush()
+
+	dbAccessor, err := db.MakeAccessor(databaseName, true, false)
+	if err != nil || dbAccessor.Handle == nil {
+		return err
+	}
+
+	return dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) error {
+		var rowsCount int64
+		err := tx.QueryRow("SELECT count(*) from catchpointkvstore").Scan(&rowsCount)
+		if err != nil {
+			return err
+		}
+
+		// ordered to make dumps more "diffable"
+		rows, err := tx.Query("SELECT key, value FROM catchpointkvstore order by key")
+		if err != nil {
+			return err
+		}
+		defer rows.Close()
+		for rows.Next() {
+			progress++
+			var key []byte
+			var value []byte
+			err := rows.Scan(&key, &value)
+			if err != nil {
+				return err
+			}
+			printKeyValue(fileWriter, key, value)
+			if time.Since(lastProgressUpdate) > 50*time.Millisecond {
+				lastProgressUpdate = time.Now()
+				printDumpingCatchpointProgressLine(int(float64(progress)*50.0/float64(rowsCount)), 50, int64(progress))
+			}
+		}
+		return nil
+	})
+}

cmd/catchpointdump/net.go

@@ -357,6 +357,11 @@ func loadAndDump(addr string, tarFile string, genesisInitState ledgercore.InitSt
 		if err != nil {
 			return err
 		}
+		err = printKeyValueStore("./ledger.tracker.sqlite", outFile)
+		if err != nil {
+			return err
+		}
+
 	}
 	return nil
 }

cmd/goal/README.md

@@ -0,0 +1,147 @@
+# Example `goal` Snippets
+
+Unless otherwise noted, it is assumed that the working directory
+begins at the top level of the `go-algorand` repo.
+
+It is also assumed that the main README's installation instructions have been followed and `make install` run so that the `goal` executable has been rebuilt from the same source as this example and is available in the shell environment.
+You can run `ls -l $(which goal)` after your `make install` and look at the installation time as a sanity check.
+
+Finally, all the `goal` commands assume that `${ALGORAND_DATA}` has been set. See the first Q/A for how this is done.
+
+## Starting a Single Node Dev Network
+
+### Q: Having just completed a new build in go-algorand, how do I get a single node dev network up, with algos in an easily accessible wallet from goal?
+
+### A:
+
+```sh
+# set this to where you want to keep the network files (and data dirs will go beneath)
+NETWORKS=~/networks
+
+# create a networks directory if you don't already have it
+mkdir -p ${NETWORKS}
+
+# set this to "name" your network
+NAME=niftynetwork
+
+# assuming here that are currently working out of the root directory of the go-algorand repo
+goal network create -n ${NAME} -r ${NETWORKS}/${NAME} -t ./test/testdata/nettemplates/OneNodeFuture.json
+
+# after the next command and for the rest of the README, we assume that `${ALGORAND_DATA}` is set
+export ALGORAND_DATA=${NETWORKS}/${NAME}/Primary
+echo $ALGORAND_DATA
+
+# start the network
+goal node start
+
+# see if it worked (run a few times, note block increasing)
+goal node status
+sleep 4  # assuming you're copy/pasting this entire block
+goal node status
+sleep 4
+goal node status
+
+# find the account with all the money
+goal account list
+
+# put it in a variable
+ACCOUNT=`goal account list | awk '{print $2}'`
+echo $ACCOUNT
+
+# send some money from the account to itself
+goal clerk send --to ${ACCOUNT} --from ${ACCOUNT} --amount 10
+```
+
+## Creating Applications
+
+### Q: How do I use goal to create an app?
+
+### A:
+Here's an example with the following assumptions:
+* all the setup is as in the first question
+* the approval program (which tests box functionality) has relative path `cmd/goal/examples/boxes.teal`
+* the clear program has relative path `cmd/goal/examples/clear.teal`
+* there are no local or global storage requirements
+
+```sh
+TEALDIR=cmd/goal/examples
+echo $TEALDIR
+
+# create the app and TAKE NOTE of its "app index"
+goal app create --creator ${ACCOUNT} --approval-prog ${TEALDIR}/boxes.teal --clear-prog ${TEALDIR}/clear.teal --global-byteslices 0 --global-ints 0 --local-byteslices 0 --local-ints 0
+```
+
+For the following questions, you'll need to use the app index. That will be shown in the last line printed. EG:
+
+```sh
+Attempting to create app (approval size 125, hash RKWO3VXBKQXF77PC6EHRLFXD4YTJYTJTGPTPWQ46YH5ESGPZ5JIA; clear size 3, hash IS4FW6ZCRMQRTDIINAVAQHD2GK6DXUNQHQ52IQGZEVPP4OEU56QA)
+Issued transaction from account ECRQFXZ7P3PLNK6QLIEVX7AXU6NTVQZHFUSEXTXMBKKOA2NTIV4PCX7XNY, txid SZK3U7AARMPQSZUICZIGYRLC7UDXJCVPV34JCBN5LIBXMF635UKA (fee 1000)
+Transaction SZK3U7AARMPQSZUICZIGYRLC7UDXJCVPV34JCBN5LIBXMF635UKA still pending as of round 12
+Transaction SZK3U7AARMPQSZUICZIGYRLC7UDXJCVPV34JCBN5LIBXMF635UKA still pending as of round 13
+Transaction SZK3U7AARMPQSZUICZIGYRLC7UDXJCVPV34JCBN5LIBXMF635UKA committed in round 14
+Created app with app index 2
+```
+
+## Funding App-Accounts
+
+### Q: How do I fund the app account so that it can satisfy its boxes min-balance requirement and allow for box creation?
+
+### A:
+Assuming you followed the previous step, and that the _app index_ is 2:
+
+```sh
+# store the app index for later usage
+APPID=2
+echo $APPID
+
+# store the app's account address into a variable
+APP_ACCOUNT=`goal app info --app-id ${APPID} | grep "Application account" | awk '{print $3}'`
+echo $APP_ACCOUNT
+
+# fund the app's account (here we're being very conservative and sending 10 algos)
+goal clerk send --to ${APP_ACCOUNT} --from ${ACCOUNT} --amount 10000000
+
+# verify the balance of the app's account
+goal account balance --address ${APP_ACCOUNT}
+```
+
+## Application Boxes in `goal`
+
+### Q: How do I use boxes in goal? In particular, I'd like to make a goal app call which:
+* accesses a particular box for a particular app
+* stores an ABI type as its contents
+
+### A:
+Here's an example with the following assumptions:
+
+* the caller's account is given by `${ACCOUNT}` (see first answer)
+* the program used is `boxes.teal` referenced above. In particular:
+  * it routes to box subroutines using the app argument at index 0 as the method signifier
+* the app id has been stored in `${APPID}` (see the previous answer)
+* the box referenced in the first non-create app-call has name `greatBox`
+* another referenced box is named `an_ABI_box`
+  * this second box is provided contents `[2,3,5]` of ABI-type `(uint8,uint8,uint8)`
+
+```sh
+# create a box with a simple non-ABI name. Note how the `--box` flag needs to be set so as to refer to the box being touched
+goal app call --from $ACCOUNT --app-id ${APPID} --box "str:greatBox" --app-arg "str:create" --app-arg "str:greatBox"
+
+# create another box
+goal app call --from ${ACCOUNT} --app-id ${APPID} --box "str:an_ABI_box" --app-arg "str:create" --app-arg "str:an_ABI_box"
+
+# set the contents to ABI type `(uint8,uint8,uint8)` with value `[2,3,5]`
+goal app call --from ${ACCOUNT} --app-id ${APPID} --box "str:an_ABI_box" --app-arg "str:set" --app-arg "str:an_ABI_box"  --app-arg "abi:(uint8,uint8,uint8):[2,3,5]"
+```
+
+### Q: How do I search for boxes in goal?
+
+### A:
+Assuming you followed the previous step to create `greatBox` and `an_ABI_box`:
+
+```sh
+# get all boxes for a given app
+goal app box list --app-id ${APPID}
+
+# get the box details for a given box
+goal app box info --app-id ${APPID} --name "str:an_ABI_box"
+```