Skip to content

fix(live): make live loader progress on a cluster with very high maxUid #7743

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 21, 2021
Merged

fix(live): make live loader progress on a cluster with very high maxUid #7743

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0433d60
update xidmap lease
NamanJain8 Apr 20, 2021
3e42361
address comments
NamanJain8 Apr 21, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion dgraph/cmd/zero/assign.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,8 @@ func (s *Server) lease(ctx context.Context, num *pb.Num) (*pb.AssignedIds, error
howMany = num.Val + leaseBandwidth
}
if howMany < num.Val || maxLease+howMany < maxLease { // check for overflow.
return &emptyAssignedIds, errors.Errorf("Cannot lease %s as the limit has reached", typ)
return &emptyAssignedIds, errors.Errorf("Cannot lease %s as the limit has reached."+
" currMax:%d", typ, s.nextLease[typ]-1)
}

var proposal pb.ZeroProposal
Expand Down
30 changes: 27 additions & 3 deletions xidmap/xidmap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ import (
"context"
"encoding/binary"
"math/rand"
"regexp"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
Expand All @@ -36,6 +39,8 @@ import (
"github.com/golang/glog"
)

var maxLeaseRegex = regexp.MustCompile(`currMax:([0-9]+)`)

// XidMapOptions specifies the options for creating a new xidmap.
type XidMapOptions struct {
UidAssigner *grpc.ClientConn
Expand Down Expand Up @@ -290,12 +295,30 @@ func (m *XidMap) updateMaxSeen(max uint64) {
// BumpTo can be used to make Zero allocate UIDs up to this given number. Attempts are made to
// ensure all future allocations of UIDs be higher than this one, but results are not guaranteed.
func (m *XidMap) BumpTo(uid uint64) {
curMax := atomic.LoadUint64(&m.maxUidSeen)
if uid <= curMax {
return
// If we have a cluster that cannot lease out new UIDs because it has already leased upto its
// max limit. Now, we try to live load the data with the given UIDs and the AssignIds complains
// that the limit has reached. Hence, update the xidmap's maxSeenUid and make progress.
updateLease := func(msg string) {
if !strings.Contains(msg, "limit has reached. currMax:") {
return
}
matches := maxLeaseRegex.FindAllStringSubmatch(msg, 1)
if len(matches) == 0 {
return
}
maxUidLeased, err := strconv.ParseUint(matches[0][1], 10, 64)
if err != nil {
glog.Errorf("While parsing currMax %+v", err)
return
}
m.updateMaxSeen(maxUidLeased)
}

for {
curMax := atomic.LoadUint64(&m.maxUidSeen)
if uid <= curMax {
return
}
glog.V(1).Infof("Bumping up to %v", uid)
num := x.Max(uid-curMax, 1e4)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
Expand All @@ -307,6 +330,7 @@ func (m *XidMap) BumpTo(uid uint64) {
m.updateMaxSeen(assigned.EndId)
return
}
updateLease(err.Error())
glog.Errorf("While requesting AssignUids(%d): %v", num, err)
if x.IsJwtExpired(err) {
if err := m.relogin(); err != nil {
Expand Down