diff --git a/ee/updatemanifest/run.go b/ee/updatemanifest/run.go deleted file mode 100644 index 8e03dbd885d..00000000000 --- a/ee/updatemanifest/run.go +++ /dev/null @@ -1,133 +0,0 @@ -// +build !oss - -/* - * Copyright 2021 Dgraph Labs, Inc. and Contributors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package updatemanifest - -import ( - "encoding/binary" - "log" - "net/url" - "os" - "strings" - - "github.com/dgraph-io/dgraph/ee" - "github.com/dgraph-io/dgraph/protos/pb" - "github.com/dgraph-io/dgraph/worker" - "github.com/dgraph-io/dgraph/x" - "github.com/pkg/errors" - "github.com/spf13/cobra" -) - -var ( - logger = log.New(os.Stderr, "", 0) - // UpdateManifest is the sub-command invoked when running "dgraph update_manifest". - UpdateManifest x.SubCommand -) - -var opt struct { - location string - key []byte -} - -func init() { - UpdateManifest.Cmd = &cobra.Command{ - Use: "update_manifest", - Short: "Run the Dgraph update tool to update the manifest from v21.03 to latest.", - Run: func(cmd *cobra.Command, args []string) { - if err := run(); err != nil { - logger.Fatalf("%v\n", err) - } - }, - Annotations: map[string]string{"group": "tool"}, - } - UpdateManifest.EnvPrefix = "DGRAPH_UPDATE_MANIFEST" - UpdateManifest.Cmd.SetHelpTemplate(x.NonRootTemplate) - - flag := UpdateManifest.Cmd.Flags() - flag.StringVarP(&opt.location, "location", "l", "", - `Sets the location of the backup. Both file URIs and s3 are supported. - This command will take care of all the full + incremental backups present in the location.`) - ee.RegisterEncFlag(flag) -} - -// Invalid bytes are replaced with the Unicode replacement rune. -// See https://golang.org/pkg/encoding/json/#Marshal -const replacementRune = rune('\ufffd') - -func parseNsAttr(attr string) (uint64, string, error) { - if strings.ContainsRune(attr, replacementRune) { - return 0, "", errors.New("replacement char found") - } - return binary.BigEndian.Uint64([]byte(attr[:8])), attr[8:], nil -} - -func run() error { - keys, err := ee.GetKeys(UpdateManifest.Conf) - if err != nil { - return err - } - opt.key = keys.EncKey - uri, err := url.Parse(opt.location) - if err != nil { - return errors.Wrapf(err, "while parsing location") - } - handler, err := worker.NewUriHandler(uri, nil) - if err != nil { - return errors.Wrapf(err, "while creating uri handler") - } - masterManifest, err := worker.GetManifestNoUpgrade(handler, uri) - if err != nil { - return errors.Wrapf(err, "while getting manifest") - } - - update := func(manifest *worker.Manifest) { - for gid, preds := range manifest.Groups { - parsedPreds := preds[:0] - for _, pred := range preds { - ns, attr, err := parseNsAttr(pred) - if err != nil { - logger.Printf("Unable to parse the pred: %v", pred) - continue - } - parsedPreds = append(parsedPreds, x.NamespaceAttr(ns, attr)) - } - manifest.Groups[gid] = parsedPreds - } - for _, op := range manifest.DropOperations { - if op.DropOp == pb.DropOperation_ATTR { - ns, attr, err := parseNsAttr(op.DropValue) - if err != nil { - logger.Printf("Unable to parse the drop operation %+v pred: %v", - op, []byte(op.DropValue)) - continue - } - op.DropValue = x.NamespaceAttr(ns, attr) - } - } - } - - // Update the master manifest with the changes for drop operations and group predicates. - for _, manifest := range masterManifest.Manifests { - if manifest.Version == 2103 { - update(manifest) - } - } - - // Rewrite the master manifest. - return errors.Wrap(worker.CreateManifest(handler, uri, masterManifest), "rewrite failed") -} diff --git a/systest/backup/filesystem/backup_test.go b/systest/backup/filesystem/backup_test.go index 2e775382f5a..b0477b86645 100644 --- a/systest/backup/filesystem/backup_test.go +++ b/systest/backup/filesystem/backup_test.go @@ -86,44 +86,6 @@ func sendRestoreRequest(t *testing.T, location string) { return } -// This test restores the old backups. -// The backup dir contains: -// - Full backup with pred "p1", "p2", "p3". (insert k1, k2, k3). -// - Incremental backup after drop data was called and "p2", "p3", "p4" inserted. --> (insert k4,k5) -// - Incremental backup after "p3" was dropped. -func TestRestoreOfOldBackup(t *testing.T) { - test := func(dir string) { - common.DirSetup(t) - common.CopyOldBackupDir(t) - - conn, err := grpc.Dial(testutil.SockAddr, - grpc.WithTransportCredentials(credentials.NewTLS(testutil.GetAlphaClientConfig(t)))) - require.NoError(t, err) - dg := dgo.NewDgraphClient(api.NewDgraphClient(conn)) - require.NoError(t, err) - - testutil.DropAll(t, dg) - time.Sleep(2 * time.Second) - - sendRestoreRequest(t, dir) - testutil.WaitForRestore(t, dg) - - queryAndCheck := func(pred string, cnt int) { - q := fmt.Sprintf(`{ me(func: has(%s)) { count(uid) } }`, pred) - r := fmt.Sprintf("{\"me\":[{\"count\":%d}]}", cnt) - resp, err := dg.NewTxn().Query(context.Background(), q) - require.NoError(t, err) - require.JSONEq(t, r, string(resp.Json)) - } - queryAndCheck("p1", 0) - queryAndCheck("p2", 2) - queryAndCheck("p3", 0) - queryAndCheck("p4", 2) - } - t.Run("backup of 20.11", func(t *testing.T) { test(oldBackupDir2) }) - t.Run("backup of 21.03", func(t *testing.T) { test(oldBackupDir3) }) -} - // This test takes a backup and then restores an old backup in a cluster incrementally. // Next, cleans up the cluster and tries restoring the backups above. // Regression test for DGRAPH-2775 @@ -192,12 +154,14 @@ func TestRestoreOfOldBackup(t *testing.T) { require.NoError(t, err) require.JSONEq(t, r, string(resp.Json)) } + queryAndCheck("p1", 0) queryAndCheck("p2", 2) queryAndCheck("p3", 0) queryAndCheck("p4", 2) } t.Run("backup of 20.11", func(t *testing.T) { test(oldBackupDir2) }) + t.Run("backup of 21.03", func(t *testing.T) { test(oldBackupDir3) }) } func TestBackupFilesystem(t *testing.T) { diff --git a/worker/backup_ee.go b/worker/backup_ee.go index aa033f90d2e..37b26b88407 100644 --- a/worker/backup_ee.go +++ b/worker/backup_ee.go @@ -562,7 +562,7 @@ func (pr *BackupProcessor) CompleteBackup(ctx context.Context, m *Manifest) erro manifest.Manifests = append(manifest.Manifests, m) if err := CreateManifest(handler, uri, manifest); err != nil { - return errors.Wrap(err, "Complete backup failed") + return errors.Wrap(err, "complete backup failed") } glog.Infof("Backup completed OK.") return nil diff --git a/worker/backup_manifest.go b/worker/backup_manifest.go index 864ea65b2b9..75f4b40d561 100644 --- a/worker/backup_manifest.go +++ b/worker/backup_manifest.go @@ -297,11 +297,6 @@ func GetManifest(h UriHandler, uri *url.URL) (*MasterManifest, error) { func CreateManifest(h UriHandler, uri *url.URL, manifest *MasterManifest) error { var err error - if !h.DirExists("./") { - if err := h.CreateDir("./"); err != nil { - return errors.Wrap(err, "createManifest failed to create path: ") - } - } w, err := h.CreateFile(tmpManifest) if err != nil { diff --git a/worker/restore_map.go b/worker/restore_map.go index 896761f1daa..9ccda63043b 100644 --- a/worker/restore_map.go +++ b/worker/restore_map.go @@ -735,15 +735,11 @@ func RunMapper(req *pb.RestoreRequest, mapDir string) (*mapResult, error) { } ns, err := strconv.ParseUint(op.DropValue, 0, 64) if err != nil { - return nil, errors.Wrap(err, "Map phase failed to parse namespace") + return nil, errors.Wrap(err, "map phase failed to parse namespace") } dropNs[ns] = struct{}{} case pb.DropOperation_ATTR: - p := op.DropValue - if manifest.Version == 0 { - p = x.NamespaceAttr(x.GalaxyNamespace, p) - } - dropAttr[p] = struct{}{} + dropAttr[op.DropValue] = struct{}{} case pb.DropOperation_NS: // pstore will be nil for export_backup tool. In that case we don't need to ban ns. if pstore == nil {