diff --git a/cmd/catchpointdump/commands.go b/cmd/catchpointdump/commands.go index e4fded7be8..e06b0b79ab 100644 --- a/cmd/catchpointdump/commands.go +++ b/cmd/catchpointdump/commands.go @@ -98,7 +98,6 @@ func main() { } if err := rootCmd.Execute(); err != nil { - fmt.Println(err) os.Exit(1) } } diff --git a/cmd/catchpointdump/file.go b/cmd/catchpointdump/file.go index b6da6029ad..ac28e775ca 100644 --- a/cmd/catchpointdump/file.go +++ b/cmd/catchpointdump/file.go @@ -33,6 +33,7 @@ import ( cmdutil "github.com/algorand/go-algorand/cmd/util" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" @@ -65,11 +66,18 @@ var fileCmd = &cobra.Command{ reportErrorf("Unable to stat '%s' : %v", tarFile, err) } tarSize := stats.Size() - if tarSize == 0 { reportErrorf("Empty file '%s' : %v", tarFile, err) } - genesisInitState := ledgercore.InitState{} + // TODO: store CurrentProtocol in catchpoint file header. + // As a temporary workaround use a current protocol version. + genesisInitState := ledgercore.InitState{ + Block: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{ + UpgradeState: bookkeeping.UpgradeState{ + CurrentProtocol: protocol.ConsensusCurrentVersion, + }, + }}, + } cfg := config.GetDefaultLocal() l, err := ledger.OpenLedger(logging.Base(), "./ledger", false, genesisInitState, cfg) if err != nil { diff --git a/cmd/catchpointdump/net.go b/cmd/catchpointdump/net.go index 9e46fea145..523560d54c 100644 --- a/cmd/catchpointdump/net.go +++ b/cmd/catchpointdump/net.go @@ -29,10 +29,12 @@ import ( "github.com/spf13/cobra" "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/data/bookkeeping" "github.com/algorand/go-algorand/ledger" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" + "github.com/algorand/go-algorand/protocol" tools "github.com/algorand/go-algorand/tools/network" "github.com/algorand/go-algorand/util" ) @@ -59,18 +61,18 @@ func init() { } var netCmd = &cobra.Command{ - Use: "net", - Short: "Download and decode (possibly all) catchpoint files from possibly all or specified the relay(s) on the network for a particular round", - Long: "Download and decode (possibly all) catchpoint files from possibly all or specified the relay(s) on the network for a particular round", - Args: validateNoPosArgsFn, - Run: func(cmd *cobra.Command, args []string) { + Use: "net", + Short: "Download and decode (possibly all) catchpoint files from possibly all or specified the relay(s) on the network for a particular round", + Long: "Download and decode (possibly all) catchpoint files from possibly all or specified the relay(s) on the network for a particular round", + Args: validateNoPosArgsFn, + SilenceUsage: true, // prevent printing usage info on error + RunE: func(cmd *cobra.Command, args []string) (err error) { if networkName == "" || round == 0 { cmd.HelpFunc()(cmd, args) - return + return fmt.Errorf("network or round not set") } var addrs []string - var err error if relayAddress != "" { addrs = []string{relayAddress} } else { @@ -81,21 +83,32 @@ var netCmd = &cobra.Command{ } for _, addr := range addrs { - tarName, err := downloadCatchpoint(addr, round) + var tarName string + tarName, err = downloadCatchpoint(addr, round) if err != nil { reportInfof("failed to download catchpoint from '%s' : %v", addr, err) continue } - err = makeFileDump(addr, tarName) + genesisInitState := ledgercore.InitState{ + Block: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{ + UpgradeState: bookkeeping.UpgradeState{ + CurrentProtocol: protocol.ConsensusCurrentVersion, + }, + }}, + } + err = makeFileDump(addr, tarName, genesisInitState) if err != nil { reportInfof("failed to make a dump from tar file for '%s' : %v", addr, err) continue } + // clear possible errors from previous run: at this point we've been succeed + err = nil if singleCatchpoint { // a catchpoint processes successfully, exit if needed break } } + return err }, } @@ -147,36 +160,48 @@ func printDownloadProgressLine(progress int, barLength int, url string, dld int6 fmt.Printf(escapeCursorUp+escapeDeleteLine+outString+" %s\n", formatSize(dld)) } -func downloadCatchpoint(addr string, round int) (tarName string, err error) { - genesisID := strings.Split(networkName, ".")[0] + "-v1.0" - url := "http://" + addr + "/v1/" + genesisID + "/ledger/" + strconv.FormatUint(uint64(round), 36) - fmt.Printf("downloading from %s\n", url) +func getRemoteDataStream(url string, hint string) (result io.ReadCloser, ctxCancel context.CancelFunc, err error) { + fmt.Printf("downloading %s from %s\n", hint, url) request, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return } - timeoutContext, timeoutContextCancel := context.WithTimeout(context.Background(), config.GetDefaultLocal().MaxCatchpointDownloadDuration) - defer timeoutContextCancel() + timeoutContext, ctxCancel := context.WithTimeout(context.Background(), config.GetDefaultLocal().MaxCatchpointDownloadDuration) request = request.WithContext(timeoutContext) network.SetUserAgentHeader(request.Header) response, err := http.DefaultClient.Do(request) if err != nil { return } - defer response.Body.Close() // check to see that we had no errors. switch response.StatusCode { case http.StatusOK: case http.StatusNotFound: // server could not find a block with that round numbers. - err = fmt.Errorf("no catchpoint file for round %d", round) + err = fmt.Errorf("no %s for round %d", hint, round) return default: err = fmt.Errorf("error response status code %d", response.StatusCode) return } + result = response.Body + return +} + +func downloadCatchpoint(addr string, round int) (tarName string, err error) { + genesisID := strings.Split(networkName, ".")[0] + "-v1.0" + urlTemplate := "http://" + addr + "/v1/" + genesisID + "/%s/" + strconv.FormatUint(uint64(round), 36) + catchpointURL := fmt.Sprintf(urlTemplate, "ledger") + + catchpointStream, catchpointCtxCancel, err := getRemoteDataStream(catchpointURL, "catchpoint") + defer catchpointCtxCancel() + if err != nil { + return + } + defer catchpointStream.Close() + dirName := "./" + strings.Split(networkName, ".")[0] + "/" + strings.Split(addr, ".")[0] os.RemoveAll(dirName) err = os.MkdirAll(dirName, 0777) @@ -193,13 +218,13 @@ func downloadCatchpoint(addr string, round int) (tarName string, err error) { }() writeChunkSize := 64 * 1024 - wdReader := util.MakeWatchdogStreamReader(response.Body, 4096, 4096, 2*time.Second) + wdReader := util.MakeWatchdogStreamReader(catchpointStream, 4096, 4096, 2*time.Second) var totalBytes int tempBytes := make([]byte, writeChunkSize) lastProgressUpdate := time.Now() progress := -25 - printDownloadProgressLine(progress, 50, url, 0) - defer printDownloadProgressLine(0, 0, url, 0) + printDownloadProgressLine(progress, 50, catchpointURL, 0) + defer printDownloadProgressLine(0, 0, catchpointURL, 0) var n int for { n, err = wdReader.Read(tempBytes) @@ -221,14 +246,13 @@ func downloadCatchpoint(addr string, round int) (tarName string, err error) { } if time.Since(lastProgressUpdate) > 50*time.Millisecond { lastProgressUpdate = time.Now() - printDownloadProgressLine(progress, 50, url, int64(totalBytes)) + printDownloadProgressLine(progress, 50, catchpointURL, int64(totalBytes)) progress++ } } } -func makeFileDump(addr string, tarFile string) error { - genesisInitState := ledgercore.InitState{} +func makeFileDump(addr string, tarFile string, genesisInitState ledgercore.InitState) error { deleteLedgerFiles := func() { os.Remove("./ledger.block.sqlite") os.Remove("./ledger.block.sqlite-shm")