Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Update Raft migrate tool to handle new proposal format #7123

Merged
merged 19 commits into from
Dec 16, 2020
Merged

chore: Update Raft migrate tool to handle new proposal format #7123

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
0c0afc6
Fix dgraph raft wal migrate tool
Dec 10, 2020
5b2f9dc
Remove older raftwal read code
Dec 11, 2020
6caebb9
Update alpha proposal entry in slice
Dec 11, 2020
5fdaf7e
Add support for ZeroProposal migration
Dec 11, 2020
a9ddc18
Fix migration of zw
ahsanbarkati Dec 12, 2020
5b46dec
fix: check entry type before conversion
NamanJain8 Dec 12, 2020
c82a90f
Merge remote-tracking branch 'origin/master' into rahulgurnani/raft-m…
NamanJain8 Dec 14, 2020
0e96441
handle zero snapshot stored in meta file correctly
NamanJain8 Dec 14, 2020
acb933f
remove unused flags and update group id
NamanJain8 Dec 14, 2020
9d48958
clean up
NamanJain8 Dec 14, 2020
571ac44
fix debug tool for new raft format
NamanJain8 Dec 14, 2020
e7221ad
cleanup
NamanJain8 Dec 14, 2020
255e2d5
address comments
NamanJain8 Dec 14, 2020
dd7ce33
do err checking on file close
NamanJain8 Dec 15, 2020
bfead41
handle empty entries emmited by raft on leader change
NamanJain8 Dec 15, 2020
4be83c2
Use MembershipState for snapshot
manishrjain Dec 15, 2020
da6613d
ignore the Proposal.Key and simplify the code
NamanJain8 Dec 15, 2020
5e26274
rename updateProposal
NamanJain8 Dec 15, 2020
fe7f2d0
store ZeroSnapshot in the zero meta file
NamanJain8 Dec 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion dgraph/cmd/debug/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -907,6 +907,7 @@ func run() {
store, err := raftwal.InitEncrypted(dir, opt.key)
x.Check(err)
fmt.Printf("RaftID: %+v\n", store.Uint(raftwal.RaftId))
isZero := store.Uint(raftwal.GroupId) == 0

// TODO: Fix the pending logic.
pending := make(map[uint64]bool)
Expand All @@ -916,7 +917,7 @@ func run() {
entries, err := store.Entries(start, last+1, 64<<20)
x.Check(err)
for _, e := range entries {
printEntry(e, pending)
printEntry(e, pending, isZero)
start = x.Max(start, e.Index)
}
}
Expand Down
28 changes: 17 additions & 11 deletions dgraph/cmd/debug/wal.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ import (
"go.etcd.io/etcd/raft/raftpb"
)

func printEntry(es raftpb.Entry, pending map[uint64]bool) {
func printEntry(es raftpb.Entry, pending map[uint64]bool, isZero bool) {
var buf bytes.Buffer
defer func() {
fmt.Printf("%s\n", buf.Bytes())
Expand All @@ -46,15 +46,21 @@ func printEntry(es raftpb.Entry, pending map[uint64]bool) {
if len(es.Data) == 0 {
return
}
var pr pb.Proposal
var zpr pb.ZeroProposal
if err := pr.Unmarshal(es.Data[8:]); err == nil {
printAlphaProposal(&buf, &pr, pending)
} else if err := zpr.Unmarshal(es.Data[8:]); err == nil {
printZeroProposal(&buf, &zpr)
var err error
if isZero {
var zpr pb.ZeroProposal
if err = zpr.Unmarshal(es.Data[8:]); err == nil {
printZeroProposal(&buf, &zpr)
return
}
} else {
fmt.Fprintf(&buf, " Unable to parse Proposal: %v", err)
var pr pb.Proposal
if err = pr.Unmarshal(es.Data[8:]); err == nil {
printAlphaProposal(&buf, &pr, pending)
return
}
}
fmt.Fprintf(&buf, " Unable to parse Proposal: %v", err)
}

type RaftStore interface {
Expand Down Expand Up @@ -116,7 +122,7 @@ func printBasic(store RaftStore) (uint64, uint64) {
return startIdx, lastIdx
}

func printRaft(db *badger.DB, store *raftmigrate.OldDiskStorage) {
func printRaft(db *badger.DB, store *raftmigrate.OldDiskStorage, groupId uint32) {
startIdx, lastIdx := printBasic(store)

commitTs := db.MaxVersion()
Expand Down Expand Up @@ -149,7 +155,7 @@ func printRaft(db *badger.DB, store *raftmigrate.OldDiskStorage) {
log.Fatalf("Unable to set data: %+v", err)
}
default:
printEntry(ent, pending)
printEntry(ent, pending, groupId == 0)
}
startIdx = x.Max(startIdx, ent.Index)
}
Expand Down Expand Up @@ -286,7 +292,7 @@ func handleWal(db *badger.DB) error {
return err

default:
printRaft(db, store)
printRaft(db, store, gid)
}
store.Closer.SignalAndWait()
}
Expand Down
Loading