Skip to content

Commit 93af817

Browse files
author
yihuang
authored
Problem: no command to fix corrupted data in versiondb (backport: #1685) (#1686)
* Problem: nixpkgs-fmt is deprecated (backport: #1677) Solution: - switch to nixfmt-rfc-style * Problem: no command to fix corrupted data in versiondb (backport: #1685) Closes: #1683 Solution: - add fix command to fix corrupted data in versiondb rename support SkipVersionZero support SkipVersionZero cleanup cleanup cleanup fix test cleanup destroy log store name fix data manually cli Update versiondb/client/fixdata.go Signed-off-by: yihuang <[email protected]> Update versiondb/client/fixdata.go Signed-off-by: yihuang <[email protected]> rnemae Update CHANGELOG.md Signed-off-by: yihuang <[email protected]> fix test don't return nil as empty slice add dryrun mode separete read from iteration add stores flag debug add timestamp api * update rocksdb * changelog * fix flag name conflict * validate timestamp * Update CHANGELOG.md Signed-off-by: yihuang <[email protected]> * skip non-zero version * flush after fix * only flush if not dry-run --------- Signed-off-by: yihuang <[email protected]>
1 parent 1f20b38 commit 93af817

File tree

15 files changed

+346
-36
lines changed

15 files changed

+346
-36
lines changed

CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66

77
* [#1670](https://github.com/crypto-org-chain/cronos/pull/1670) Fix state overwrite in debug trace APIs.
88
* [#1679](https://github.com/crypto-org-chain/cronos/pull/1679) Include no trace detail on insufficient balance fix.
9+
* [#1685](https://github.com/crypto-org-chain/cronos/pull/1685) Add command to fix versiondb corrupted data.
10+
* [#1688](https://github.com/crypto-org-chain/cronos/pull/1688) Add timestamp api to versiondb iterator.
11+
* [#1686](https://github.com/crypto-org-chain/cronos/pull/1686) Update rocksdb to 9.7.4.
912

1013
*Oct 14, 2024*
1114

app/versiondb.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ func (app *App) setupVersionDB(
2323
if err := os.MkdirAll(dataDir, os.ModePerm); err != nil {
2424
return nil, err
2525
}
26+
2627
versionDB, err := tsrocksdb.NewStore(dataDir)
2728
if err != nil {
2829
return nil, err
@@ -34,6 +35,9 @@ func (app *App) setupVersionDB(
3435
exposeStoreKeys = append(exposeStoreKeys, storeKey)
3536
}
3637

38+
// see: https://github.com/crypto-org-chain/cronos/issues/1683
39+
versionDB.SetSkipVersionZero(true)
40+
3741
service := versiondb.NewStreamingService(versionDB, exposeStoreKeys)
3842
app.SetStreamingService(service)
3943

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ require (
2727
github.com/golang/protobuf v1.5.4
2828
github.com/gorilla/mux v1.8.0
2929
github.com/grpc-ecosystem/grpc-gateway v1.16.0
30-
github.com/linxGnu/grocksdb v1.9.2
30+
github.com/linxGnu/grocksdb v1.9.7
3131
github.com/peggyjv/gravity-bridge/module/v2 v2.0.0-20220420162017-838c0d25e974
3232
github.com/spf13/cast v1.6.0
3333
github.com/spf13/cobra v1.8.0

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1085,8 +1085,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
10851085
github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
10861086
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
10871087
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
1088-
github.com/linxGnu/grocksdb v1.9.2 h1:O3mzvO0wuzQ9mtlHbDrShixyVjVbmuqTjFrzlf43wZ8=
1089-
github.com/linxGnu/grocksdb v1.9.2/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA=
1088+
github.com/linxGnu/grocksdb v1.9.7 h1:Bp2r1Yti/IXxEobZZnDooXAui/Q+5gVqgQMenLWyDUw=
1089+
github.com/linxGnu/grocksdb v1.9.7/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA=
10901090
github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM=
10911091
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
10921092
github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=

gomod2nix.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -403,8 +403,8 @@ schema = 3
403403
version = "v0.1.0"
404404
hash = "sha256-wQqGTtRWsfR9n0O/SXHVgECebbnNmHddxJIbG63OJBQ="
405405
[mod."github.com/linxGnu/grocksdb"]
406-
version = "v1.9.2"
407-
hash = "sha256-ThXtaXx6LvRIFW4xLHsMrVWdsN2qobLPA0InLmlADOM="
406+
version = "v1.9.7"
407+
hash = "sha256-ZSomnYZRo7gHB9/FW55MebNkNzn0DuR96RfsVpAwjIQ="
408408
[mod."github.com/magiconair/properties"]
409409
version = "v1.8.7"
410410
hash = "sha256-XQ2bnc2s7/IH3WxEO4GishZurMyKwEclZy1DXg+2xXc="

nix/rocksdb.nix

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,13 @@
2121

2222
stdenv.mkDerivation (finalAttrs: {
2323
pname = "rocksdb";
24-
version = "9.2.1";
24+
version = "9.7.4";
2525

2626
src = fetchFromGitHub {
2727
owner = "facebook";
2828
repo = finalAttrs.pname;
2929
rev = "v${finalAttrs.version}";
30-
hash = "sha256-Zifn5Gu/4h6TaEqSaWQ2mFdryeAarqbHWW3fKUGGFac=";
30+
hash = "sha256-u5uuShM2SxHc9/zL4UU56IhCcR/ZQbzde0LgOYS44bM=";
3131
};
3232

3333
nativeBuildInputs = [

versiondb/client/cmd.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ func ChangeSetGroupCmd(opts Options) *cobra.Command {
2828
ChangeSetToVersionDBCmd(),
2929
RestoreAppDBCmd(opts),
3030
RestoreVersionDBCmd(),
31+
FixDataCmd(opts.DefaultStores),
3132
)
3233
return cmd
3334
}

versiondb/client/fixdata.go

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package client
2+
3+
import (
4+
"github.com/crypto-org-chain/cronos/versiondb/tsrocksdb"
5+
"github.com/linxGnu/grocksdb"
6+
"github.com/spf13/cobra"
7+
)
8+
9+
const (
10+
FlagDryRun = "dry-run"
11+
FlagStore = "store-name"
12+
)
13+
14+
func FixDataCmd(defaultStores []string) *cobra.Command {
15+
cmd := &cobra.Command{
16+
Use: "fixdata <dir>",
17+
Args: cobra.ExactArgs(1),
18+
Short: "Fix wrong data in versiondb, see: https://github.com/crypto-org-chain/cronos/issues/1683",
19+
RunE: func(cmd *cobra.Command, args []string) error {
20+
dir := args[0]
21+
dryRun, err := cmd.Flags().GetBool(FlagDryRun)
22+
if err != nil {
23+
return err
24+
}
25+
stores, err := cmd.Flags().GetStringArray(FlagStore)
26+
if err != nil {
27+
return err
28+
}
29+
if len(stores) == 0 {
30+
stores = defaultStores
31+
}
32+
33+
var (
34+
db *grocksdb.DB
35+
cfHandle *grocksdb.ColumnFamilyHandle
36+
)
37+
38+
if dryRun {
39+
db, cfHandle, err = tsrocksdb.OpenVersionDBForReadOnly(dir, false)
40+
} else {
41+
db, cfHandle, err = tsrocksdb.OpenVersionDB(dir)
42+
}
43+
if err != nil {
44+
return err
45+
}
46+
47+
versionDB := tsrocksdb.NewStoreWithDB(db, cfHandle)
48+
if err := versionDB.FixData(stores, dryRun); err != nil {
49+
return err
50+
}
51+
52+
return nil
53+
},
54+
}
55+
56+
cmd.Flags().Bool(FlagDryRun, false, "Dry run, do not write to the database, open the database in read-only mode.")
57+
cmd.Flags().StringArray(FlagStore, []string{}, "Store names to fix, if not specified, all stores will be fixed.")
58+
return cmd
59+
}

versiondb/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ require (
1414
github.com/cosmos/iavl v0.21.0-alpha.1.0.20230904092046-df3db2d96583
1515
github.com/crypto-org-chain/cronos/memiavl v0.0.3
1616
github.com/golang/snappy v0.0.4
17-
github.com/linxGnu/grocksdb v1.9.2
17+
github.com/linxGnu/grocksdb v1.9.7
1818
github.com/spf13/cast v1.5.0
1919
github.com/spf13/cobra v1.6.1
2020
github.com/stretchr/testify v1.9.0

versiondb/go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -515,8 +515,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
515515
github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
516516
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
517517
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
518-
github.com/linxGnu/grocksdb v1.9.2 h1:O3mzvO0wuzQ9mtlHbDrShixyVjVbmuqTjFrzlf43wZ8=
519-
github.com/linxGnu/grocksdb v1.9.2/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA=
518+
github.com/linxGnu/grocksdb v1.9.7 h1:Bp2r1Yti/IXxEobZZnDooXAui/Q+5gVqgQMenLWyDUw=
519+
github.com/linxGnu/grocksdb v1.9.7/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA=
520520
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
521521
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
522522
github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=

0 commit comments

Comments
 (0)