From 86be5e680ed81351d0792529db75b9b6a1878a54 Mon Sep 17 00:00:00 2001 From: Brian Olson Date: Tue, 26 Nov 2019 09:38:26 -0500 Subject: [PATCH 01/13] add ?raw=1 to local block api to return msgpack bytes with full data --- daemon/algod/api/server/v1/handlers/errors.go | 1 + .../algod/api/server/v1/handlers/handlers.go | 34 +++++++++++++++++++ rpcs/httpFetcher.go | 2 +- rpcs/ledgerService.go | 15 ++++---- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/daemon/algod/api/server/v1/handlers/errors.go b/daemon/algod/api/server/v1/handlers/errors.go index 98dede9ae7..17eeca5305 100644 --- a/daemon/algod/api/server/v1/handlers/errors.go +++ b/daemon/algod/api/server/v1/handlers/errors.go @@ -24,6 +24,7 @@ var ( errFailedRetrievingNodeStatus = "failed retrieving node status" errFailedRetrievingAsset = "failed to retrieve asset information" errFailedParsingRoundNumber = "failed to parse the round number" + errFailedParsingRawOption = "failed to parse the raw option" errFailedParsingMaxAssetsToList = "failed to parse max assets, must be between %d and %d" errFailedParsingAssetIdx = "failed to parse asset index" errFailedToGetAssetCreator = "failed to retrieve asset creator from the ledger" diff --git a/daemon/algod/api/server/v1/handlers/handlers.go b/daemon/algod/api/server/v1/handlers/handlers.go index dfbb9cb866..d00058241d 100644 --- a/daemon/algod/api/server/v1/handlers/handlers.go +++ b/daemon/algod/api/server/v1/handlers/handlers.go @@ -39,6 +39,7 @@ import ( "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/node" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/rpcs" ) func nodeStatus(node *node.AlgorandFullNode) (res v1.NodeStatus, err error) { @@ -1179,6 +1180,12 @@ func GetBlock(ctx lib.ReqContext, w http.ResponseWriter, r *http.Request) { // minimum: 0 // required: true // description: The round from which to fetch block information. + // - name: raw + // in: query + // type: integer + // format: int64 + // required: false + // description: Return raw msgpack block bytes // Responses: // 200: // "$ref": '#/responses/BlockResponse' @@ -1196,6 +1203,33 @@ func GetBlock(ctx lib.ReqContext, w http.ResponseWriter, r *http.Request) { return } + // raw msgpack option: + rawstr := r.FormValue("raw") + if rawstr != "" { + rawint, err := strconv.ParseUint(rawstr, 10, 64) + if err != nil { + lib.ErrorResponse(w, http.StatusBadRequest, err, errFailedParsingRawOption, ctx.Log) + return + } + if rawint != 0 { + blockbytes, err := rpcs.RawBlockBytes(ctx.Node.Ledger(), basics.Round(queryRound)) + if err != nil { + lib.ErrorResponse(w, http.StatusInternalServerError, err, errFailedLookingUpLedger, ctx.Log) + return + } + w.Header().Set("Content-Type", rpcs.LedgerResponseContentType) + w.Header().Set("Content-Length", strconv.Itoa(len(blockbytes))) + w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") + w.WriteHeader(http.StatusOK) + _, err = w.Write(blockbytes) + if err != nil { + ctx.Log.Warnf("algod failed to write an object to the response stream: %v", err) + } + return + } + } + + // decoded json-reencoded default: ledger := ctx.Node.Ledger() b, c, err := ledger.BlockCert(basics.Round(queryRound)) if err != nil { diff --git a/rpcs/httpFetcher.go b/rpcs/httpFetcher.go index 4f32a7db34..b5c339cd3b 100644 --- a/rpcs/httpFetcher.go +++ b/rpcs/httpFetcher.go @@ -104,7 +104,7 @@ func (hf *HTTPFetcher) GetBlockBytes(ctx context.Context, r basics.Round) (data // TODO: Temporarily allow old and new content types so we have time for lazy upgrades // Remove this 'old' string after next release. const ledgerResponseContentTypeOld = "application/algorand-block-v1" - if contentTypes[0] != ledgerResponseContentType && contentTypes[0] != ledgerResponseContentTypeOld { + if contentTypes[0] != LedgerResponseContentType && contentTypes[0] != ledgerResponseContentTypeOld { hf.log.Warnf("http block fetcher response has an invalid content type : %s", contentTypes[0]) response.Body.Close() return nil, fmt.Errorf("http block fetcher invalid content type '%s'", contentTypes[0]) diff --git a/rpcs/ledgerService.go b/rpcs/ledgerService.go index a67690b1bc..5d0a3298ad 100644 --- a/rpcs/ledgerService.go +++ b/rpcs/ledgerService.go @@ -36,7 +36,8 @@ import ( "github.com/algorand/go-algorand/protocol" ) -const ledgerResponseContentType = "application/x-algorand-block-v1" +// LedgerResponseContentType is the HTTP Content-Type header for a raw binary block +const LedgerResponseContentType = "application/x-algorand-block-v1" const ledgerResponseHasBlockCacheControl = "public, max-age=31536000, immutable" // 31536000 seconds are one year. const ledgerResponseMissingBlockCacheControl = "public, max-age=1, must-revalidate" // cache for 1 second, and force revalidation afterward const ledgerServerMaxBodyLength = 512 // we don't really pass meaningful content here, so 512 bytes should be a safe limit @@ -160,7 +161,7 @@ func (ls *LedgerService) ServeHTTP(response http.ResponseWriter, request *http.R response.WriteHeader(http.StatusBadRequest) return } - encodedBlockCert, err := ls.encodedBlockCert(round) + encodedBlockCert, err := RawBlockBytes(ls.ledger, basics.Round(round)) if err != nil { switch err.(type) { case ledger.ErrNoEntry: @@ -176,7 +177,7 @@ func (ls *LedgerService) ServeHTTP(response http.ResponseWriter, request *http.R } } - response.Header().Set("Content-Type", ledgerResponseContentType) + response.Header().Set("Content-Type", LedgerResponseContentType) response.Header().Set("Content-Length", strconv.Itoa(len(encodedBlockCert))) response.Header().Set("Cache-Control", ledgerResponseHasBlockCacheControl) response.WriteHeader(http.StatusOK) @@ -236,7 +237,8 @@ func (ls *LedgerService) handleCatchupReq(ctx context.Context, reqMsg network.In return } res.Round = req.Round - encodedBlob, err := ls.encodedBlockCert(req.Round) + encodedBlob, err := RawBlockBytes(ls.ledger, basics.Round(req.Round)) + if err != nil { res.Error = err.Error() return @@ -254,8 +256,9 @@ func (ls *LedgerService) sendCatchupRes(ctx context.Context, target network.Unic } } -func (ls *LedgerService) encodedBlockCert(round uint64) ([]byte, error) { - blk, cert, err := ls.ledger.EncodedBlockCert(basics.Round(round)) +// RawBlockBytes return the msgpack bytes for a block +func RawBlockBytes(ledger *data.Ledger, round basics.Round) ([]byte, error) { + blk, cert, err := ledger.EncodedBlockCert(round) if err != nil { return nil, err } From 09bd96eacff85b6a525ec941e525923529ecf3f6 Mon Sep 17 00:00:00 2001 From: Max Justicz Date: Wed, 11 Dec 2019 11:51:25 -0500 Subject: [PATCH 02/13] add goal ledger rawblock cmd --- cmd/goal/ledger.go | 31 +++++++++++++ daemon/algod/api/client/restClient.go | 46 ++++++++++++++++--- .../algod/api/server/v1/handlers/responses.go | 12 +++++ daemon/algod/api/spec/v1/model.go | 13 ++++++ libgoal/libgoal.go | 9 ++++ 5 files changed, 105 insertions(+), 6 deletions(-) diff --git a/cmd/goal/ledger.go b/cmd/goal/ledger.go index e36b6b8490..a8f24d6622 100644 --- a/cmd/goal/ledger.go +++ b/cmd/goal/ledger.go @@ -22,8 +22,18 @@ import ( "github.com/spf13/cobra" ) +var ( + round uint64 + rawFilename string +) + func init() { ledgerCmd.AddCommand(supplyCmd) + ledgerCmd.AddCommand(rawBlockCmd) + + rawBlockCmd.Flags().Uint64VarP(&round, "round", "r", 0, "The round to fetch the raw block for") + rawBlockCmd.Flags().StringVarP(&rawFilename, "out", "o", "-", "The filename to dump the raw block to (if not set, use stdout)") + rawBlockCmd.MarkFlagRequired("round") } var ledgerCmd = &cobra.Command{ @@ -52,3 +62,24 @@ var supplyCmd = &cobra.Command{ fmt.Printf("Round: %v\nTotal Money: %v microAlgos\nOnline Money: %v microAlgos\n", response.Round, response.TotalMoney, response.OnlineMoney) }, } + +var rawBlockCmd = &cobra.Command{ + Use: "rawblock", + Short: "Print the raw, encoded msgpack block to stdout", + Long: "Print the raw, encoded msgpack block to stdout", + Args: validateNoPosArgsFn, + Run: func(cmd *cobra.Command, _ []string) { + dataDir := ensureSingleDataDir() + client := ensureAlgodClient(dataDir) + response, err := client.RawBlock(round) + if err != nil { + reportErrorf(errorRequestFail, err) + } + + // If rawFilename flag was not set, the default value '-' will write to stdout + err = writeFile(rawFilename, response, 0600) + if err != nil { + reportErrorf(fileWriteError, rawFilename, err) + } + }, +} diff --git a/daemon/algod/api/client/restClient.go b/daemon/algod/api/client/restClient.go index 5894b797bf..5e1e7cd35d 100644 --- a/daemon/algod/api/client/restClient.go +++ b/daemon/algod/api/client/restClient.go @@ -39,6 +39,7 @@ const ( authHeader = "X-Algo-API-Token" healthCheckEndpoint = "/health" apiVersionPathPrefix = "/v1" + maxRawResponseBytes = 50e6 ) // unversionedPaths ais a set of paths that should not be prefixed by the API version @@ -87,7 +88,7 @@ func stripTransaction(tx string) string { } // submitForm is a helper used for submitting (ex.) GETs and POSTs to the server -func (client RestClient) submitForm(response interface{}, path string, request interface{}, requestMethod string, encodeJSON bool) error { +func (client RestClient) submitForm(response interface{}, path string, request interface{}, requestMethod string, encodeJSON bool, decodeJSON bool) error { var err error queryURL := client.serverURL queryURL.Path = path @@ -134,11 +135,12 @@ func (client RestClient) submitForm(response interface{}, path string, request i httpClient := &http.Client{} resp, err := httpClient.Do(req) - if err != nil { return err } + // Ensure response isn't too large + resp.Body = http.MaxBytesReader(nil, resp.Body, maxRawResponseBytes) defer resp.Body.Close() err = extractError(resp) @@ -146,20 +148,42 @@ func (client RestClient) submitForm(response interface{}, path string, request i return err } - dec := json.NewDecoder(resp.Body) - return dec.Decode(&response) + if decodeJSON { + dec := json.NewDecoder(resp.Body) + return dec.Decode(&response) + } + + // Response must implement RawResponse + raw, ok := response.(v1.RawResponse) + if !ok { + return fmt.Errorf("can only decode raw response into type implementing v1.RawResponse") + } + + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + raw.SetBytes(bodyBytes) + return nil } // get performs a GET request to the specific path against the server func (client RestClient) get(response interface{}, path string, request interface{}) error { - return client.submitForm(response, path, request, "GET", false /* encodeJSON */) + return client.submitForm(response, path, request, "GET", false /* encodeJSON */, true /* decodeJSON */) +} + +// getRaw behaves identically to get but doesn't json decode the response, and +// the response must implement the v1.RawResponse interface +func (client RestClient) getRaw(response v1.RawResponse, path string, request interface{}) error { + return client.submitForm(response, path, request, "GET", false /* encodeJSON */, false /* decodeJSON */) } // post sends a POST request to the given path with the given request object. // No query parameters will be sent if request is nil. // response must be a pointer to an object as post writes the response there. func (client RestClient) post(response interface{}, path string, request interface{}) error { - return client.submitForm(response, path, request, "POST", true /* encodeJSON */) + return client.submitForm(response, path, request, "POST", true /* encodeJSON */, true /* decodeJSON */) } // Status retrieves the StatusResponse from the running node @@ -219,6 +243,10 @@ type assetsParams struct { Max uint64 `url:"max"` } +type rawblockParams struct { + Raw uint64 `url:"raw"` +} + // TransactionsByAddr returns all transactions for a PK [addr] in the [first, // last] rounds range. func (client RestClient) TransactionsByAddr(addr string, first, last, max uint64) (response v1.TransactionList, err error) { @@ -303,6 +331,12 @@ func (client RestClient) Block(round uint64) (response v1.Block, err error) { return } +// RawBlock gets the encoded, raw msgpack block for the given round +func (client RestClient) RawBlock(round uint64) (response v1.RawBlock, err error) { + err = client.getRaw(&response, fmt.Sprintf("/block/%d", round), rawblockParams{1}) + return +} + // GetGoRoutines gets a dump of the goroutines from pprof // Not supported func (client RestClient) GetGoRoutines(ctx context.Context) (goRoutines string, err error) { diff --git a/daemon/algod/api/server/v1/handlers/responses.go b/daemon/algod/api/server/v1/handlers/responses.go index fab747f38c..b41b30bd99 100644 --- a/daemon/algod/api/server/v1/handlers/responses.go +++ b/daemon/algod/api/server/v1/handlers/responses.go @@ -175,6 +175,18 @@ func (r TransactionParamsResponse) getBody() interface{} { return r.Body } +// RawBlockResponse contains encoded, raw block information +// +// swagger:response RawBlockResponse +type RawBlockResponse struct { + // in: body + Body *v1.RawBlock +} + +func (r RawBlockResponse) getBody() interface{} { + return r.Body +} + // BlockResponse contains block information // // swagger:response BlockResponse diff --git a/daemon/algod/api/spec/v1/model.go b/daemon/algod/api/spec/v1/model.go index 157c48e469..a6fa62bc9a 100644 --- a/daemon/algod/api/spec/v1/model.go +++ b/daemon/algod/api/spec/v1/model.go @@ -608,6 +608,19 @@ type TransactionParams struct { MinTxnFee uint64 `json:"minFee"` } +type RawResponse interface { + SetBytes([]byte) +} + +// RawBlock represents an encoded msgpack block +// swagger:model RawBlock +// swagger:strfmt byte +type RawBlock []byte + +func (rb *RawBlock) SetBytes(b []byte) { + *rb = b +} + // Block contains a block information // swagger:model Block type Block struct { diff --git a/libgoal/libgoal.go b/libgoal/libgoal.go index d51726caac..6760e7d12b 100644 --- a/libgoal/libgoal.go +++ b/libgoal/libgoal.go @@ -672,6 +672,15 @@ func (c *Client) Block(round uint64) (resp v1.Block, err error) { return } +// RawBlock takes a round and returns its block +func (c *Client) RawBlock(round uint64) (resp v1.RawBlock, err error) { + algod, err := c.ensureAlgodClient() + if err == nil { + resp, err = algod.RawBlock(round) + } + return +} + // HealthCheck returns an error if something is wrong func (c *Client) HealthCheck() error { algod, err := c.ensureAlgodClient() From 196e3dfd3cd0ce95ab50bb3a466552f8389ad174 Mon Sep 17 00:00:00 2001 From: Max Justicz Date: Wed, 11 Dec 2019 11:52:20 -0500 Subject: [PATCH 03/13] update helptext --- cmd/goal/ledger.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/goal/ledger.go b/cmd/goal/ledger.go index a8f24d6622..283a25f0a2 100644 --- a/cmd/goal/ledger.go +++ b/cmd/goal/ledger.go @@ -65,8 +65,8 @@ var supplyCmd = &cobra.Command{ var rawBlockCmd = &cobra.Command{ Use: "rawblock", - Short: "Print the raw, encoded msgpack block to stdout", - Long: "Print the raw, encoded msgpack block to stdout", + Short: "Dump the raw, encoded msgpack block to a file or stdout", + Long: "Dump the raw, encoded msgpack block to a file or stdout", Args: validateNoPosArgsFn, Run: func(cmd *cobra.Command, _ []string) { dataDir := ensureSingleDataDir() From efbaabdb3269d100ef37243b82c393ebb588789c Mon Sep 17 00:00:00 2001 From: Max Justicz Date: Wed, 11 Dec 2019 11:59:04 -0500 Subject: [PATCH 04/13] use stdoutFilenameValue instead of hardcoded value --- cmd/goal/ledger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/goal/ledger.go b/cmd/goal/ledger.go index 283a25f0a2..03b92cebad 100644 --- a/cmd/goal/ledger.go +++ b/cmd/goal/ledger.go @@ -32,7 +32,7 @@ func init() { ledgerCmd.AddCommand(rawBlockCmd) rawBlockCmd.Flags().Uint64VarP(&round, "round", "r", 0, "The round to fetch the raw block for") - rawBlockCmd.Flags().StringVarP(&rawFilename, "out", "o", "-", "The filename to dump the raw block to (if not set, use stdout)") + rawBlockCmd.Flags().StringVarP(&rawFilename, "out", "o", stdoutFilenameValue, "The filename to dump the raw block to (if not set, use stdout)") rawBlockCmd.MarkFlagRequired("round") } From b77f110e61345419c4882a56068b8378d9a3d77e Mon Sep 17 00:00:00 2001 From: Max Justicz Date: Wed, 11 Dec 2019 12:42:57 -0500 Subject: [PATCH 05/13] fix lint ugh --- daemon/algod/api/spec/v1/model.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/daemon/algod/api/spec/v1/model.go b/daemon/algod/api/spec/v1/model.go index a6fa62bc9a..bad26a1d21 100644 --- a/daemon/algod/api/spec/v1/model.go +++ b/daemon/algod/api/spec/v1/model.go @@ -608,6 +608,7 @@ type TransactionParams struct { MinTxnFee uint64 `json:"minFee"` } +// RawResponse is fulfilled by responses that should not be decoded as msgpack type RawResponse interface { SetBytes([]byte) } @@ -617,6 +618,7 @@ type RawResponse interface { // swagger:strfmt byte type RawBlock []byte +// SetBytes fulfills the RawResponse interface on RawBlock func (rb *RawBlock) SetBytes(b []byte) { *rb = b } From a602a9e636a65e4f16e16bff3b30db39c6ee3ba0 Mon Sep 17 00:00:00 2001 From: Max Justicz Date: Wed, 11 Dec 2019 14:02:15 -0500 Subject: [PATCH 06/13] make round arg positional --- cmd/goal/ledger.go | 13 ++++++++----- cmd/goal/messages.go | 3 +++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cmd/goal/ledger.go b/cmd/goal/ledger.go index 03b92cebad..92408b0e46 100644 --- a/cmd/goal/ledger.go +++ b/cmd/goal/ledger.go @@ -18,12 +18,12 @@ package main import ( "fmt" + "strconv" "github.com/spf13/cobra" ) var ( - round uint64 rawFilename string ) @@ -31,9 +31,7 @@ func init() { ledgerCmd.AddCommand(supplyCmd) ledgerCmd.AddCommand(rawBlockCmd) - rawBlockCmd.Flags().Uint64VarP(&round, "round", "r", 0, "The round to fetch the raw block for") rawBlockCmd.Flags().StringVarP(&rawFilename, "out", "o", stdoutFilenameValue, "The filename to dump the raw block to (if not set, use stdout)") - rawBlockCmd.MarkFlagRequired("round") } var ledgerCmd = &cobra.Command{ @@ -67,8 +65,13 @@ var rawBlockCmd = &cobra.Command{ Use: "rawblock", Short: "Dump the raw, encoded msgpack block to a file or stdout", Long: "Dump the raw, encoded msgpack block to a file or stdout", - Args: validateNoPosArgsFn, - Run: func(cmd *cobra.Command, _ []string) { + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + round, err := strconv.ParseUint(args[0], 10, 64) + if err != nil { + reportErrorf(errParsingRoundNumber, err) + } + dataDir := ensureSingleDataDir() client := ensureAlgodClient(dataDir) response, err := client.RawBlock(round) diff --git a/cmd/goal/messages.go b/cmd/goal/messages.go index ddeb675dcc..9145a017b5 100644 --- a/cmd/goal/messages.go +++ b/cmd/goal/messages.go @@ -146,4 +146,7 @@ const ( errWalletNotFound = "Wallet '%s' not found" errDefaultWalletNotFound = "Wallet with ID '%s' not found. Was the default wallet deleted?" errGettingToken = "Couldn't get token for wallet '%s' (ID: %s): %s" + + // Ledger + errParsingRoundNumber = "Error parsing round number: %s" ) From 5cb33df3fa9a44a094cdc38f5a5aef98ec77f86c Mon Sep 17 00:00:00 2001 From: Max Justicz Date: Wed, 11 Dec 2019 14:18:12 -0500 Subject: [PATCH 07/13] update use string --- cmd/goal/ledger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/goal/ledger.go b/cmd/goal/ledger.go index 92408b0e46..0d13993b4c 100644 --- a/cmd/goal/ledger.go +++ b/cmd/goal/ledger.go @@ -62,7 +62,7 @@ var supplyCmd = &cobra.Command{ } var rawBlockCmd = &cobra.Command{ - Use: "rawblock", + Use: "rawblock [round number]", Short: "Dump the raw, encoded msgpack block to a file or stdout", Long: "Dump the raw, encoded msgpack block to a file or stdout", Args: cobra.ExactArgs(1), From 3ad97dabc5308bbeefca3a9717e6a0930ab4e265 Mon Sep 17 00:00:00 2001 From: Max Justicz Date: Thu, 12 Dec 2019 10:55:16 -0500 Subject: [PATCH 08/13] dump encoded by default, move transcode to its own package --- cmd/goal/ledger.go | 39 +++++++++++++++---- cmd/goal/messages.go | 4 +- cmd/msgpacktool/main.go | 4 +- .../transcode}/core.go | 22 +++++------ .../transcode}/core_test.go | 4 +- 5 files changed, 51 insertions(+), 22 deletions(-) rename {cmd/msgpacktool => protocol/transcode}/core.go (90%) rename {cmd/msgpacktool => protocol/transcode}/core_test.go (99%) diff --git a/cmd/goal/ledger.go b/cmd/goal/ledger.go index 0d13993b4c..7b15714cd7 100644 --- a/cmd/goal/ledger.go +++ b/cmd/goal/ledger.go @@ -17,21 +17,30 @@ package main import ( + "bytes" "fmt" "strconv" "github.com/spf13/cobra" + + "github.com/algorand/go-algorand/protocol/transcode" ) var ( - rawFilename string + rawFilename string + rawBlock bool + base32Encoding bool + strictJSON bool ) func init() { ledgerCmd.AddCommand(supplyCmd) - ledgerCmd.AddCommand(rawBlockCmd) + ledgerCmd.AddCommand(blockCmd) - rawBlockCmd.Flags().StringVarP(&rawFilename, "out", "o", stdoutFilenameValue, "The filename to dump the raw block to (if not set, use stdout)") + blockCmd.Flags().StringVarP(&rawFilename, "out", "o", stdoutFilenameValue, "The filename to dump the raw block to (if not set, use stdout)") + blockCmd.Flags().BoolVarP(&rawBlock, "raw", "r", false, "Dump raw block as msgpack") + blockCmd.Flags().BoolVar(&base32Encoding, "b32", false, "Encode binary blobs using base32 instead of base64") + blockCmd.Flags().BoolVar(&strictJSON, "strict", false, "Strict JSON decode: turn all keys into strings") } var ledgerCmd = &cobra.Command{ @@ -61,10 +70,10 @@ var supplyCmd = &cobra.Command{ }, } -var rawBlockCmd = &cobra.Command{ - Use: "rawblock [round number]", - Short: "Dump the raw, encoded msgpack block to a file or stdout", - Long: "Dump the raw, encoded msgpack block to a file or stdout", +var blockCmd = &cobra.Command{ + Use: "block [round number]", + Short: "Dump a block to a file or stdout", + Long: "Dump a block to a file or stdout", Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { round, err := strconv.ParseUint(args[0], 10, 64) @@ -79,6 +88,22 @@ var rawBlockCmd = &cobra.Command{ reportErrorf(errorRequestFail, err) } + // Unless the user asked for the raw block, + // print the msgpack decoded version + if !rawBlock { + in := bytes.NewBuffer(response) + out := bytes.NewBuffer(nil) + err = transcode.Transcode(true, in, out, base32Encoding, strictJSON) + if err != nil { + reportErrorf(errEncodingBlockAsJSON, err) + } + response = out.Bytes() + } else { + if base32Encoding || strictJSON { + reportErrorf(errBadBlockArgs) + } + } + // If rawFilename flag was not set, the default value '-' will write to stdout err = writeFile(rawFilename, response, 0600) if err != nil { diff --git a/cmd/goal/messages.go b/cmd/goal/messages.go index 9145a017b5..529cf0b1b3 100644 --- a/cmd/goal/messages.go +++ b/cmd/goal/messages.go @@ -148,5 +148,7 @@ const ( errGettingToken = "Couldn't get token for wallet '%s' (ID: %s): %s" // Ledger - errParsingRoundNumber = "Error parsing round number: %s" + errParsingRoundNumber = "Error parsing round number: %s" + errBadBlockArgs = "Cannot combine --b32=true or --strict=true with --raw" + errEncodingBlockAsJSON = "Error encoding block as json: %s" ) diff --git a/cmd/msgpacktool/main.go b/cmd/msgpacktool/main.go index c836293ba2..1d5e2b580b 100644 --- a/cmd/msgpacktool/main.go +++ b/cmd/msgpacktool/main.go @@ -45,6 +45,8 @@ import ( "flag" "fmt" "os" + + "github.com/algorand/go-algorand/protocol/transcode" ) var mpToJSON = flag.Bool("d", false, "Decode msgpack to JSON") @@ -64,7 +66,7 @@ func main() { os.Exit(1) } - err := transcode(*mpToJSON, os.Stdin, os.Stdout) + err := transcode.Transcode(*mpToJSON, os.Stdin, os.Stdout, *base32Encoding, *strictJSON) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) diff --git a/cmd/msgpacktool/core.go b/protocol/transcode/core.go similarity index 90% rename from cmd/msgpacktool/core.go rename to protocol/transcode/core.go index 3ebc52fcb4..5de4d2f44d 100644 --- a/cmd/msgpacktool/core.go +++ b/protocol/transcode/core.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . -package main +package transcode import ( "encoding/base32" @@ -32,7 +32,7 @@ type decoder interface { Decode(v interface{}) error } -func transcode(mpToJSON bool, in io.ReadCloser, out io.WriteCloser) error { +func Transcode(mpToJSON bool, in io.Reader, out io.Writer, base32Encoding, strictJSON bool) error { canonicalMsgpackHandle := new(codec.MsgpackHandle) canonicalMsgpackHandle.ErrorIfNoField = true canonicalMsgpackHandle.ErrorIfNoArrayExpand = true @@ -62,9 +62,6 @@ func transcode(mpToJSON bool, in io.ReadCloser, out io.WriteCloser) error { enc = codec.NewEncoder(out, canonicalMsgpackHandle) } - defer in.Close() - defer out.Close() - for { var a interface{} err := dec.Decode(&a) @@ -77,7 +74,7 @@ func transcode(mpToJSON bool, in io.ReadCloser, out io.WriteCloser) error { } if mpToJSON { - a = toJSON(a) + a = toJSON(a, base32Encoding, strictJSON) } else { a = fromJSON(a) } @@ -93,7 +90,7 @@ func transcode(mpToJSON bool, in io.ReadCloser, out io.WriteCloser) error { } } -func toJSON(a interface{}) interface{} { +func toJSON(a interface{}, base32Encoding, strictJSON bool) interface{} { switch v := a.(type) { case map[interface{}]interface{}: r := make(map[interface{}]interface{}) @@ -106,16 +103,18 @@ func toJSON(a interface{}) interface{} { eb, ok2 := e.([]byte) if ok1 && ok2 { - if *base32Encoding { + if base32Encoding { r[fmt.Sprintf("%s:b32", ks)] = base32.StdEncoding.EncodeToString(eb) } else { r[fmt.Sprintf("%s:b64", ks)] = base64.StdEncoding.EncodeToString(eb) } } else { - if *strictJSON { + if strictJSON { k = fmt.Sprintf("%v", k) } - r[toJSON(k)] = toJSON(e) + kenc := toJSON(k, base32Encoding, strictJSON) + eenc := toJSON(e, base32Encoding, strictJSON) + r[kenc] = eenc } } return r @@ -123,7 +122,8 @@ func toJSON(a interface{}) interface{} { case []interface{}: r := make([]interface{}, 0) for _, e := range v { - r = append(r, toJSON(e)) + eenc := toJSON(e, base32Encoding, strictJSON) + r = append(r, eenc) } return r diff --git a/cmd/msgpacktool/core_test.go b/protocol/transcode/core_test.go similarity index 99% rename from cmd/msgpacktool/core_test.go rename to protocol/transcode/core_test.go index a6f703a8b7..60fa658327 100644 --- a/cmd/msgpacktool/core_test.go +++ b/protocol/transcode/core_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with go-algorand. If not, see . -package main +package transcode import ( "encoding/base32" @@ -30,7 +30,7 @@ import ( ) func transcodeNoError(t *testing.T, mpToJSON bool, in io.ReadCloser, out io.WriteCloser) { - err := transcode(mpToJSON, in, out) + err := Transcode(mpToJSON, in, out) require.NoError(t, err) } From dfe818ff0820928b9c3484aab6543015c993667d Mon Sep 17 00:00:00 2001 From: Max Justicz Date: Thu, 12 Dec 2019 10:58:12 -0500 Subject: [PATCH 09/13] move args, fix lint --- cmd/goal/ledger.go | 2 +- cmd/msgpacktool/main.go | 2 +- protocol/transcode/core.go | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/goal/ledger.go b/cmd/goal/ledger.go index 7b15714cd7..3812f5a9ef 100644 --- a/cmd/goal/ledger.go +++ b/cmd/goal/ledger.go @@ -93,7 +93,7 @@ var blockCmd = &cobra.Command{ if !rawBlock { in := bytes.NewBuffer(response) out := bytes.NewBuffer(nil) - err = transcode.Transcode(true, in, out, base32Encoding, strictJSON) + err = transcode.Transcode(true, base32Encoding, strictJSON, in, out) if err != nil { reportErrorf(errEncodingBlockAsJSON, err) } diff --git a/cmd/msgpacktool/main.go b/cmd/msgpacktool/main.go index 1d5e2b580b..a2983cbe11 100644 --- a/cmd/msgpacktool/main.go +++ b/cmd/msgpacktool/main.go @@ -66,7 +66,7 @@ func main() { os.Exit(1) } - err := transcode.Transcode(*mpToJSON, os.Stdin, os.Stdout, *base32Encoding, *strictJSON) + err := transcode.Transcode(*mpToJSON, *base32Encoding, *strictJSON, os.Stdin, os.Stdout) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) diff --git a/protocol/transcode/core.go b/protocol/transcode/core.go index 5de4d2f44d..90c9866fba 100644 --- a/protocol/transcode/core.go +++ b/protocol/transcode/core.go @@ -32,7 +32,8 @@ type decoder interface { Decode(v interface{}) error } -func Transcode(mpToJSON bool, in io.Reader, out io.Writer, base32Encoding, strictJSON bool) error { +// Transcode turns msgpack to JSON or JSON to msgpack +func Transcode(mpToJSON bool, base32Encoding, strictJSON bool, in io.Reader, out io.Writer) error { canonicalMsgpackHandle := new(codec.MsgpackHandle) canonicalMsgpackHandle.ErrorIfNoField = true canonicalMsgpackHandle.ErrorIfNoArrayExpand = true From cbf902255a3601093b7097819061d143fe811d77 Mon Sep 17 00:00:00 2001 From: Max Justicz Date: Thu, 12 Dec 2019 10:59:37 -0500 Subject: [PATCH 10/13] fix comment --- cmd/goal/ledger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/goal/ledger.go b/cmd/goal/ledger.go index 3812f5a9ef..e9f177e851 100644 --- a/cmd/goal/ledger.go +++ b/cmd/goal/ledger.go @@ -89,7 +89,7 @@ var blockCmd = &cobra.Command{ } // Unless the user asked for the raw block, - // print the msgpack decoded version + // print the block encoded as JSON if !rawBlock { in := bytes.NewBuffer(response) out := bytes.NewBuffer(nil) From 17b10eaada7d0fae086a7ea0a3c039b3fa25779d Mon Sep 17 00:00:00 2001 From: Max Justicz Date: Thu, 12 Dec 2019 12:37:34 -0500 Subject: [PATCH 11/13] address feedback --- cmd/goal/ledger.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/goal/ledger.go b/cmd/goal/ledger.go index e9f177e851..f387cc50d8 100644 --- a/cmd/goal/ledger.go +++ b/cmd/goal/ledger.go @@ -27,7 +27,7 @@ import ( ) var ( - rawFilename string + blockFilename string rawBlock bool base32Encoding bool strictJSON bool @@ -37,8 +37,8 @@ func init() { ledgerCmd.AddCommand(supplyCmd) ledgerCmd.AddCommand(blockCmd) - blockCmd.Flags().StringVarP(&rawFilename, "out", "o", stdoutFilenameValue, "The filename to dump the raw block to (if not set, use stdout)") - blockCmd.Flags().BoolVarP(&rawBlock, "raw", "r", false, "Dump raw block as msgpack") + blockCmd.Flags().StringVarP(&blockFilename, "out", "o", stdoutFilenameValue, "The filename to dump the block to (if not set, use stdout)") + blockCmd.Flags().BoolVarP(&rawBlock, "raw", "r", false, "Format block as msgpack") blockCmd.Flags().BoolVar(&base32Encoding, "b32", false, "Encode binary blobs using base32 instead of base64") blockCmd.Flags().BoolVar(&strictJSON, "strict", false, "Strict JSON decode: turn all keys into strings") } @@ -104,10 +104,10 @@ var blockCmd = &cobra.Command{ } } - // If rawFilename flag was not set, the default value '-' will write to stdout - err = writeFile(rawFilename, response, 0600) + // If blockFilename flag was not set, the default value '-' will write to stdout + err = writeFile(blockFilename, response, 0600) if err != nil { - reportErrorf(fileWriteError, rawFilename, err) + reportErrorf(fileWriteError, blockFilename, err) } }, } From 69ca5e4cc0aa10d235d55f766d868e4b06a4f740 Mon Sep 17 00:00:00 2001 From: Max Justicz Date: Thu, 12 Dec 2019 12:39:15 -0500 Subject: [PATCH 12/13] fix test --- protocol/transcode/core_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol/transcode/core_test.go b/protocol/transcode/core_test.go index 60fa658327..127a8c536f 100644 --- a/protocol/transcode/core_test.go +++ b/protocol/transcode/core_test.go @@ -30,7 +30,7 @@ import ( ) func transcodeNoError(t *testing.T, mpToJSON bool, in io.ReadCloser, out io.WriteCloser) { - err := Transcode(mpToJSON, in, out) + err := Transcode(mpToJSON, false, false, in, out) require.NoError(t, err) } From 65f0ea08994f36756c805661a7afff76430f8283 Mon Sep 17 00:00:00 2001 From: Max Justicz Date: Thu, 12 Dec 2019 17:29:00 -0500 Subject: [PATCH 13/13] try to fix transcode test --- protocol/transcode/core_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/protocol/transcode/core_test.go b/protocol/transcode/core_test.go index 127a8c536f..01ffa4dc8f 100644 --- a/protocol/transcode/core_test.go +++ b/protocol/transcode/core_test.go @@ -30,6 +30,8 @@ import ( ) func transcodeNoError(t *testing.T, mpToJSON bool, in io.ReadCloser, out io.WriteCloser) { + defer in.Close() + defer out.Close() err := Transcode(mpToJSON, false, false, in, out) require.NoError(t, err) }