Skip to content

tools: update catchpointdump 'database' and 'database check' commands to handle staging tables and KVs #4802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cce merged 3 commits into from
Nov 18, 2022
Merged

tools: update catchpointdump 'database' and 'database check' commands to handle staging tables and KVs #4802

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0f853bb
update catchpointdump 'database' and 'database check' commands to han…
cce Nov 16, 2022
ea7cfd8
Merge remote-tracking branch 'origin/master' into fix-catchpointdump
cce Nov 16, 2022
a4e3c6b
address CR comments
cce Nov 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions cmd/catchpointdump/database.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,16 @@ import (
)

var ledgerTrackerFilename string
var ledgerTrackerStaging bool

func init() {
databaseCmd.Flags().StringVarP(&ledgerTrackerFilename, "tracker", "t", "", "Specify the ledger tracker file name ( i.e. ./ledger.tracker.sqlite )")
databaseCmd.Flags().StringVarP(&outFileName, "output", "o", "", "Specify an outfile for the dump ( i.e. ledger.dump.txt )")
databaseCmd.Flags().BoolVarP(&ledgerTrackerStaging, "staging", "s", false, "Specify whether to look in the catchpoint staging or regular tables. (default false)")
databaseCmd.AddCommand(checkCmd)

checkCmd.Flags().StringVarP(&ledgerTrackerFilename, "tracker", "t", "", "Specify the ledger tracker file name ( i.e. ./ledger.tracker.sqlite )")
checkCmd.Flags().BoolVarP(&ledgerTrackerStaging, "staging", "s", false, "Specify whether to look in the catchpoint staging or regular tables. (default false)")
}

var databaseCmd = &cobra.Command{
Expand All @@ -58,10 +61,14 @@ var databaseCmd = &cobra.Command{
}
defer outFile.Close()
}
err = printAccountsDatabase(ledgerTrackerFilename, ledger.CatchpointFileHeader{}, outFile, nil)
err = printAccountsDatabase(ledgerTrackerFilename, ledgerTrackerStaging, ledger.CatchpointFileHeader{}, outFile, nil)
if err != nil {
reportErrorf("Unable to print account database : %v", err)
}
err = printKeyValueStore(ledgerTrackerFilename, ledgerTrackerStaging, ledger.CatchpointFileHeader{}, outFile)
if err != nil {
reportErrorf("Unable to print key value store : %v", err)
}
},
}

Expand Down Expand Up @@ -99,14 +106,19 @@ func checkDatabase(databaseName string, outFile *os.File) error {

var stats merkletrie.Stats
err = dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
committer, err := ledger.MakeMerkleCommitter(tx, false)
committer, err := ledger.MakeMerkleCommitter(tx, ledgerTrackerStaging)
if err != nil {
return err
}
trie, err := merkletrie.MakeTrie(committer, ledger.TrieMemoryConfig)
if err != nil {
return err
}
root, err := trie.RootHash()
fmt.Fprintf(outFile, " Root: %s\n", root.String())
if err != nil {
return err
}

@jannotti jannotti Nov 17, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does printing the root even when err != nil make sense?

Also, doesn't String() get called implicitly by %s?

@cce cce Nov 18, 2022

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated

stats, err = trie.GetStats()
if err != nil {
return err
Expand Down
85 changes: 27 additions & 58 deletions cmd/catchpointdump/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,11 @@ var fileCmd = &cobra.Command{
defer outFile.Close()
}

err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, excludedFields.GetSlice())
err = printAccountsDatabase("./ledger.tracker.sqlite", true, fileHeader, outFile, excludedFields.GetSlice())
if err != nil {
reportErrorf("Unable to print account database : %v", err)
}
err = printKeyValueStore("./ledger.tracker.sqlite", outFile)
err = printKeyValueStore("./ledger.tracker.sqlite", true, fileHeader, outFile)
if err != nil {
reportErrorf("Unable to print key value store : %v", err)
}
Expand Down Expand Up @@ -246,7 +246,7 @@ func printDumpingCatchpointProgressLine(progress int, barLength int, dld int64)
fmt.Printf(escapeCursorUp + escapeDeleteLine + outString + "\n")
}

func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFileHeader, outFile *os.File, excludeFields []string) error {
func printAccountsDatabase(databaseName string, stagingTables bool, fileHeader ledger.CatchpointFileHeader, outFile *os.File, excludeFields []string) error {
lastProgressUpdate := time.Now()
progress := uint64(0)
defer printDumpingCatchpointProgressLine(0, 0, 0)
Expand Down Expand Up @@ -322,6 +322,9 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
if fileHeader.Version == 0 {
var totals ledgercore.AccountTotals
id := ""
if stagingTables {
id = "catchpointStaging"
}
row := tx.QueryRow("SELECT online, onlinerewardunits, offline, offlinerewardunits, notparticipating, notparticipatingrewardunits, rewardslevel FROM accounttotals WHERE id=?", id)
err = row.Scan(&totals.Online.Money.Raw, &totals.Online.RewardUnits,
&totals.Offline.Money.Raw, &totals.Offline.RewardUnits,
Expand All @@ -339,7 +342,7 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile

balancesTable := "accountbase"
resourcesTable := "resources"
if fileHeader.Version != 0 {
if stagingTables {
balancesTable = "catchpointbalances"
resourcesTable = "catchpointresources"
}
Expand All @@ -365,60 +368,21 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
return nil
}

if fileHeader.Version < ledger.CatchpointFileVersionV6 {
var rows *sql.Rows
rows, err = tx.Query(fmt.Sprintf("SELECT address, data FROM %s order by address", balancesTable))
acctCount := 0
acctCb := func(addr basics.Address, data basics.AccountData) {
err = printer(addr, data, progress)
if err != nil {
return
}
defer rows.Close()

for rows.Next() {
var addrbuf []byte
var buf []byte
err = rows.Scan(&addrbuf, &buf)
if err != nil {
return
}
Comment thread
algorandskiy marked this conversation as resolved.

var addr basics.Address
if len(addrbuf) != len(addr) {
err = fmt.Errorf("account DB address length mismatch: %d != %d", len(addrbuf), len(addr))
return
}
copy(addr[:], addrbuf)

var data basics.AccountData
err = protocol.Decode(buf, &data)
if err != nil {
return
}

err = printer(addr, data, progress)
if err != nil {
return
}

progress++
}
err = rows.Err()
} else {
acctCount := 0
acctCb := func(addr basics.Address, data basics.AccountData) {
err = printer(addr, data, progress)
if err != nil {
return
}
progress++
acctCount++
}
_, err = ledger.LoadAllFullAccounts(context.Background(), tx, balancesTable, resourcesTable, acctCb)
if err != nil {
return
}
if acctCount != int(rowsCount) {
return fmt.Errorf("expected %d accounts but got only %d", rowsCount, acctCount)
}
progress++
acctCount++
}
_, err = ledger.LoadAllFullAccounts(context.Background(), tx, balancesTable, resourcesTable, acctCb)
if err != nil {
return
}
if acctCount != int(rowsCount) {
return fmt.Errorf("expected %d accounts but got only %d", rowsCount, acctCount)
}

// increase the deadline warning to disable the warning message.
Expand All @@ -439,7 +403,7 @@ func printKeyValue(writer *bufio.Writer, key, value []byte) {
fmt.Fprintf(writer, "%s : %v\n", pretty, base64.StdEncoding.EncodeToString(value))
}

func printKeyValueStore(databaseName string, outFile *os.File) error {
func printKeyValueStore(databaseName string, stagingTables bool, fileHeader ledger.CatchpointFileHeader, outFile *os.File) error {

@jannotti jannotti Nov 17, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you just add fileHeader for consistency among the print funcs? It's unused, isn't it?

@michaeldiamant michaeldiamant Nov 17, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I confirm fileHeader is unused. I do not recall why the parameter was added during debugging. Unless @cce recalls, I think it's preferable to remove it.

@cce cce Nov 18, 2022

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, just removed it, it was left over from before I added the stagingTables arg.

fmt.Printf("\n")
printDumpingCatchpointProgressLine(0, 50, 0)
lastProgressUpdate := time.Now()
Expand All @@ -454,15 +418,20 @@ func printKeyValueStore(databaseName string, outFile *os.File) error {
return err
}

kvTable := "kvstore"
if stagingTables {
kvTable = "catchpointkvstore"
}

return dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) error {
var rowsCount int64
err := tx.QueryRow("SELECT count(*) from catchpointkvstore").Scan(&rowsCount)
err := tx.QueryRow(fmt.Sprintf("SELECT count(*) from %s", kvTable)).Scan(&rowsCount)
if err != nil {
return err
}

// ordered to make dumps more "diffable"
rows, err := tx.Query("SELECT key, value FROM catchpointkvstore order by key")
rows, err := tx.Query(fmt.Sprintf("SELECT key, value FROM %s order by key", kvTable))
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/catchpointdump/net.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,11 +353,11 @@ func loadAndDump(addr string, tarFile string, genesisInitState ledgercore.InitSt
return err
}
defer outFile.Close()
err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, excludedFields.GetSlice())
err = printAccountsDatabase("./ledger.tracker.sqlite", true, fileHeader, outFile, excludedFields.GetSlice())
if err != nil {
return err
}
err = printKeyValueStore("./ledger.tracker.sqlite", outFile)
err = printKeyValueStore("./ledger.tracker.sqlite", true, fileHeader, outFile)
if err != nil {
return err
}
Expand Down