Skip to content

tools: update catchpointdump 'database' and 'database check' commands to handle staging tables and KVs #4802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cce merged 3 commits into from
Nov 18, 2022
Merged

tools: update catchpointdump 'database' and 'database check' commands to handle staging tables and KVs #4802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0f853bb
update catchpointdump 'database' and 'database check' commands to han…
cce Nov 16, 2022
ea7cfd8
Merge remote-tracking branch 'origin/master' into fix-catchpointdump
cce Nov 16, 2022
a4e3c6b
address CR comments
cce Nov 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions cmd/catchpointdump/database.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,16 @@ import (
)

var ledgerTrackerFilename string
var ledgerTrackerStaging bool

func init() {
databaseCmd.Flags().StringVarP(&ledgerTrackerFilename, "tracker", "t", "", "Specify the ledger tracker file name ( i.e. ./ledger.tracker.sqlite )")
databaseCmd.Flags().StringVarP(&outFileName, "output", "o", "", "Specify an outfile for the dump ( i.e. ledger.dump.txt )")
databaseCmd.Flags().BoolVarP(&ledgerTrackerStaging, "staging", "s", false, "Specify whether to look in the catchpoint staging or regular tables. (default false)")
databaseCmd.AddCommand(checkCmd)

checkCmd.Flags().StringVarP(&ledgerTrackerFilename, "tracker", "t", "", "Specify the ledger tracker file name ( i.e. ./ledger.tracker.sqlite )")
checkCmd.Flags().BoolVarP(&ledgerTrackerStaging, "staging", "s", false, "Specify whether to look in the catchpoint staging or regular tables. (default false)")
}

var databaseCmd = &cobra.Command{
Expand All @@ -58,10 +61,14 @@ var databaseCmd = &cobra.Command{
}
defer outFile.Close()
}
err = printAccountsDatabase(ledgerTrackerFilename, ledger.CatchpointFileHeader{}, outFile, nil)
err = printAccountsDatabase(ledgerTrackerFilename, ledgerTrackerStaging, ledger.CatchpointFileHeader{}, outFile, nil)
if err != nil {
reportErrorf("Unable to print account database : %v", err)
}
err = printKeyValueStore(ledgerTrackerFilename, ledgerTrackerStaging, outFile)
if err != nil {
reportErrorf("Unable to print key value store : %v", err)
}
},
}

Expand Down Expand Up @@ -99,14 +106,19 @@ func checkDatabase(databaseName string, outFile *os.File) error {

var stats merkletrie.Stats
err = dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
committer, err := ledger.MakeMerkleCommitter(tx, false)
committer, err := ledger.MakeMerkleCommitter(tx, ledgerTrackerStaging)
if err != nil {
return err
}
trie, err := merkletrie.MakeTrie(committer, ledger.TrieMemoryConfig)
if err != nil {
return err
}
root, err := trie.RootHash()
if err != nil {
return err
}
fmt.Fprintf(outFile, " Root: %s\n", root)
stats, err = trie.GetStats()
if err != nil {
return err
Expand Down
25 changes: 16 additions & 9 deletions cmd/catchpointdump/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,11 @@ var fileCmd = &cobra.Command{
defer outFile.Close()
}

err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, excludedFields.GetSlice())
err = printAccountsDatabase("./ledger.tracker.sqlite", true, fileHeader, outFile, excludedFields.GetSlice())
if err != nil {
reportErrorf("Unable to print account database : %v", err)
}
err = printKeyValueStore("./ledger.tracker.sqlite", outFile)
err = printKeyValueStore("./ledger.tracker.sqlite", true, outFile)
if err != nil {
reportErrorf("Unable to print key value store : %v", err)
}
Expand Down Expand Up @@ -246,7 +246,7 @@ func printDumpingCatchpointProgressLine(progress int, barLength int, dld int64)
fmt.Printf(escapeCursorUp + escapeDeleteLine + outString + "\n")
}

func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFileHeader, outFile *os.File, excludeFields []string) error {
func printAccountsDatabase(databaseName string, stagingTables bool, fileHeader ledger.CatchpointFileHeader, outFile *os.File, excludeFields []string) error {
lastProgressUpdate := time.Now()
progress := uint64(0)
defer printDumpingCatchpointProgressLine(0, 0, 0)
Expand Down Expand Up @@ -322,6 +322,9 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
if fileHeader.Version == 0 {
var totals ledgercore.AccountTotals
id := ""
if stagingTables {
id = "catchpointStaging"
}
row := tx.QueryRow("SELECT online, onlinerewardunits, offline, offlinerewardunits, notparticipating, notparticipatingrewardunits, rewardslevel FROM accounttotals WHERE id=?", id)
err = row.Scan(&totals.Online.Money.Raw, &totals.Online.RewardUnits,
&totals.Offline.Money.Raw, &totals.Offline.RewardUnits,
Expand All @@ -339,7 +342,7 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile

balancesTable := "accountbase"
resourcesTable := "resources"
if fileHeader.Version != 0 {
if stagingTables {
balancesTable = "catchpointbalances"
resourcesTable = "catchpointresources"
}
Expand All @@ -365,7 +368,7 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
return nil
}

if fileHeader.Version < ledger.CatchpointFileVersionV6 {
if fileHeader.Version != 0 && fileHeader.Version < ledger.CatchpointFileVersionV6 {
var rows *sql.Rows
rows, err = tx.Query(fmt.Sprintf("SELECT address, data FROM %s order by address", balancesTable))
if err != nil {
Expand Down Expand Up @@ -420,7 +423,6 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
return fmt.Errorf("expected %d accounts but got only %d", rowsCount, acctCount)
}
}

// increase the deadline warning to disable the warning message.
_, _ = db.ResetTransactionWarnDeadline(ctx, tx, time.Now().Add(5*time.Second))
return err
Expand All @@ -439,7 +441,7 @@ func printKeyValue(writer *bufio.Writer, key, value []byte) {
fmt.Fprintf(writer, "%s : %v\n", pretty, base64.StdEncoding.EncodeToString(value))
}

func printKeyValueStore(databaseName string, outFile *os.File) error {
func printKeyValueStore(databaseName string, stagingTables bool, outFile *os.File) error {
fmt.Printf("\n")
printDumpingCatchpointProgressLine(0, 50, 0)
lastProgressUpdate := time.Now()
Expand All @@ -454,15 +456,20 @@ func printKeyValueStore(databaseName string, outFile *os.File) error {
return err
}

kvTable := "kvstore"
if stagingTables {
kvTable = "catchpointkvstore"
}

return dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) error {
var rowsCount int64
err := tx.QueryRow("SELECT count(*) from catchpointkvstore").Scan(&rowsCount)
err := tx.QueryRow(fmt.Sprintf("SELECT count(*) from %s", kvTable)).Scan(&rowsCount)
if err != nil {
return err
}

// ordered to make dumps more "diffable"
rows, err := tx.Query("SELECT key, value FROM catchpointkvstore order by key")
rows, err := tx.Query(fmt.Sprintf("SELECT key, value FROM %s order by key", kvTable))
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/catchpointdump/net.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,11 +353,11 @@ func loadAndDump(addr string, tarFile string, genesisInitState ledgercore.InitSt
return err
}
defer outFile.Close()
err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, excludedFields.GetSlice())
err = printAccountsDatabase("./ledger.tracker.sqlite", true, fileHeader, outFile, excludedFields.GetSlice())
if err != nil {
return err
}
err = printKeyValueStore("./ledger.tracker.sqlite", outFile)
err = printKeyValueStore("./ledger.tracker.sqlite", true, outFile)
if err != nil {
return err
}
Expand Down