diff --git a/worker/online_restore_ee.go b/worker/online_restore_ee.go index cf229a1a1b7..77f4c6985d6 100644 --- a/worker/online_restore_ee.go +++ b/worker/online_restore_ee.go @@ -62,10 +62,38 @@ func ProcessRestoreRequest(ctx context.Context, req *pb.RestoreRequest) (int, er if err := VerifyBackup(req, &creds, currentGroups); err != nil { return 0, errors.Wrapf(err, "failed to verify backup") } - if err := FillRestoreCredentials(req.Location, req); err != nil { return 0, errors.Wrapf(err, "cannot fill restore proposal with the right credentials") } + // Restore Tracker keeps tracks of ongoing restore operation initiated on the node. + // Restore Tracker are node specific. They manage restore operation initiated on the node. + // If already initiated on this node, rt.Add() will return already running operation error. + restoreId, err := rt.Add() + if err != nil { + return 0, errors.Wrapf(err, "cannot assign ID to restore operation") + } + + // This check if any restore operation running on the node. + // Operation initiated on other nodes doesn't have record in the record tracker. + // This keeps track if there is an already running restore operation return the error. + // IMP: This introduces few corner cases. + // Like two concurrent restore operation on different nodes. + // Considering Restore as admin operation, solving all those complexities has low gains + // than to sacrifice the simplicity. + isRestoreRunning := func() bool { + tasks := GetOngoingTasks() + for _, t := range tasks { + if t == opRestore.String() { + return true + } + } + return false + } + if isRestoreRunning() { + return 0, errors.Errorf("another restore operation is already running. " + + "Please retry later.") + } + req.RestoreTs = State.GetTimestamp(false) // TODO: prevent partial restores when proposeRestoreOrSend only sends the restore @@ -80,10 +108,6 @@ func ProcessRestoreRequest(ctx context.Context, req *pb.RestoreRequest) (int, er }() } - restoreId, err := rt.Add() - if err != nil { - return 0, errors.Wrapf(err, "cannot assign ID to restore operation") - } go func(restoreId int) { errs := make([]error, 0) for range currentGroups {