catchpointdump: set genesis proto version

cmd/catchpointdump/commands.go

@@ -98,7 +98,6 @@ func main() {
 	}
 
 	if err := rootCmd.Execute(); err != nil {
-		fmt.Println(err)
 		os.Exit(1)
 	}
 }

cmd/catchpointdump/file.go

@@ -33,6 +33,7 @@ import (
 	cmdutil "github.com/algorand/go-algorand/cmd/util"
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/ledger"
 	"github.com/algorand/go-algorand/ledger/ledgercore"
 	"github.com/algorand/go-algorand/logging"
@@ -65,11 +66,18 @@ var fileCmd = &cobra.Command{
 			reportErrorf("Unable to stat '%s' : %v", tarFile, err)
 		}
 		tarSize := stats.Size()
-
 		if tarSize == 0 {
 			reportErrorf("Empty file '%s' : %v", tarFile, err)
 		}
-		genesisInitState := ledgercore.InitState{}
+		// TODO: store CurrentProtocol in catchpoint file header.
+		// As a temporary workaround use a current protocol version.
+		genesisInitState := ledgercore.InitState{
+			Block: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{
+				UpgradeState: bookkeeping.UpgradeState{
+					CurrentProtocol: protocol.ConsensusCurrentVersion,
+				},
+			}},
+		}
 		cfg := config.GetDefaultLocal()
 		l, err := ledger.OpenLedger(logging.Base(), "./ledger", false, genesisInitState, cfg)
 		if err != nil {

cmd/catchpointdump/net.go

@@ -29,10 +29,12 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/algorand/go-algorand/config"
+	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/ledger"
 	"github.com/algorand/go-algorand/ledger/ledgercore"
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/network"
+	"github.com/algorand/go-algorand/protocol"
 	tools "github.com/algorand/go-algorand/tools/network"
 	"github.com/algorand/go-algorand/util"
 )
@@ -59,18 +61,18 @@ func init() {
 }
 
 var netCmd = &cobra.Command{
-	Use:   "net",
-	Short: "Download and decode (possibly all) catchpoint files from possibly all or specified the relay(s) on the network for a particular round",
-	Long:  "Download and decode (possibly all) catchpoint files from possibly all or specified the relay(s) on the network for a particular round",
-	Args:  validateNoPosArgsFn,
-	Run: func(cmd *cobra.Command, args []string) {
+	Use:          "net",
+	Short:        "Download and decode (possibly all) catchpoint files from possibly all or specified the relay(s) on the network for a particular round",
+	Long:         "Download and decode (possibly all) catchpoint files from possibly all or specified the relay(s) on the network for a particular round",
+	Args:         validateNoPosArgsFn,
+	SilenceUsage: true, // prevent printing usage info on error
+	RunE: func(cmd *cobra.Command, args []string) (err error) {
 		if networkName == "" || round == 0 {
 			cmd.HelpFunc()(cmd, args)
-			return
+			return fmt.Errorf("network or round not set")
 		}
 
 		var addrs []string
-		var err error
 		if relayAddress != "" {
 			addrs = []string{relayAddress}
 		} else {
@@ -81,21 +83,32 @@ var netCmd = &cobra.Command{
 		}
 
 		for _, addr := range addrs {
-			tarName, err := downloadCatchpoint(addr, round)
+			var tarName string
+			tarName, err = downloadCatchpoint(addr, round)
 			if err != nil {
 				reportInfof("failed to download catchpoint from '%s' : %v", addr, err)
 				continue
 			}
-			err = makeFileDump(addr, tarName)
+			genesisInitState := ledgercore.InitState{
+				Block: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{
+					UpgradeState: bookkeeping.UpgradeState{
+						CurrentProtocol: protocol.ConsensusCurrentVersion,
+					},
+				}},
+			}
+			err = makeFileDump(addr, tarName, genesisInitState)
 			if err != nil {
 				reportInfof("failed to make a dump from tar file for '%s' : %v", addr, err)
 				continue
 			}
+			// clear possible errors from previous run: at this point we've been succeed
+			err = nil
 			if singleCatchpoint {
 				// a catchpoint processes successfully, exit if needed
 				break
 			}
 		}
+		return err
 	},
 }
 
@@ -147,36 +160,48 @@ func printDownloadProgressLine(progress int, barLength int, url string, dld int6
 	fmt.Printf(escapeCursorUp+escapeDeleteLine+outString+" %s\n", formatSize(dld))
 }
 
-func downloadCatchpoint(addr string, round int) (tarName string, err error) {
-	genesisID := strings.Split(networkName, ".")[0] + "-v1.0"
-	url := "http://" + addr + "/v1/" + genesisID + "/ledger/" + strconv.FormatUint(uint64(round), 36)
-	fmt.Printf("downloading from %s\n", url)
+func getRemoteDataStream(url string, hint string) (result io.ReadCloser, ctxCancel context.CancelFunc, err error) {
+	fmt.Printf("downloading %s from %s\n", hint, url)
 	request, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
 		return
 	}
 
-	timeoutContext, timeoutContextCancel := context.WithTimeout(context.Background(), config.GetDefaultLocal().MaxCatchpointDownloadDuration)
-	defer timeoutContextCancel()
+	timeoutContext, ctxCancel := context.WithTimeout(context.Background(), config.GetDefaultLocal().MaxCatchpointDownloadDuration)
 	request = request.WithContext(timeoutContext)
 	network.SetUserAgentHeader(request.Header)
 	response, err := http.DefaultClient.Do(request)
 	if err != nil {
 		return
 	}
-	defer response.Body.Close()
 
 	// check to see that we had no errors.
 	switch response.StatusCode {
 	case http.StatusOK:
 	case http.StatusNotFound: // server could not find a block with that round numbers.
-		err = fmt.Errorf("no catchpoint file for round %d", round)
+		err = fmt.Errorf("no %s for round %d", hint, round)
 		return
 	default:
 		err = fmt.Errorf("error response status code %d", response.StatusCode)
 		return
 	}
 
+	result = response.Body
+	return
+}
+
+func downloadCatchpoint(addr string, round int) (tarName string, err error) {
+	genesisID := strings.Split(networkName, ".")[0] + "-v1.0"
+	urlTemplate := "http://" + addr + "/v1/" + genesisID + "/%s/" + strconv.FormatUint(uint64(round), 36)
+	catchpointURL := fmt.Sprintf(urlTemplate, "ledger")
+
+	catchpointStream, catchpointCtxCancel, err := getRemoteDataStream(catchpointURL, "catchpoint")
+	defer catchpointCtxCancel()
+	if err != nil {
+		return
+	}
+	defer catchpointStream.Close()
+
 	dirName := "./" + strings.Split(networkName, ".")[0] + "/" + strings.Split(addr, ".")[0]
 	os.RemoveAll(dirName)
 	err = os.MkdirAll(dirName, 0777)
@@ -193,13 +218,13 @@ func downloadCatchpoint(addr string, round int) (tarName string, err error) {
 	}()
 	writeChunkSize := 64 * 1024
 
-	wdReader := util.MakeWatchdogStreamReader(response.Body, 4096, 4096, 2*time.Second)
+	wdReader := util.MakeWatchdogStreamReader(catchpointStream, 4096, 4096, 2*time.Second)
 	var totalBytes int
 	tempBytes := make([]byte, writeChunkSize)
 	lastProgressUpdate := time.Now()
 	progress := -25
-	printDownloadProgressLine(progress, 50, url, 0)
-	defer printDownloadProgressLine(0, 0, url, 0)
+	printDownloadProgressLine(progress, 50, catchpointURL, 0)
+	defer printDownloadProgressLine(0, 0, catchpointURL, 0)
 	var n int
 	for {
 		n, err = wdReader.Read(tempBytes)
@@ -221,14 +246,13 @@ func downloadCatchpoint(addr string, round int) (tarName string, err error) {
 		}
 		if time.Since(lastProgressUpdate) > 50*time.Millisecond {
 			lastProgressUpdate = time.Now()
-			printDownloadProgressLine(progress, 50, url, int64(totalBytes))
+			printDownloadProgressLine(progress, 50, catchpointURL, int64(totalBytes))
 			progress++
 		}
 	}
 }
 
-func makeFileDump(addr string, tarFile string) error {
-	genesisInitState := ledgercore.InitState{}
+func makeFileDump(addr string, tarFile string, genesisInitState ledgercore.InitState) error {
 	deleteLedgerFiles := func() {
 		os.Remove("./ledger.block.sqlite")
 		os.Remove("./ledger.block.sqlite-shm")