chore(upgrade): update the upgrade tool for CORS (depreciated predica…

…tes) change (#7486)

Some of the internal predicates have been removed in #7431 and #7451.
When restoring from a backup taken on a version older than 21.03, we should skip restoring them.
Also, its export will fail in data loading(bulk/live load).

Instead of polluting the backup code with the restore fixation logic, we will use the dgraph upgrade tool to do the migration.
When upgrading from version 20.11 to 21.03, the changes of this PR will be applied. It will update the graphql schema with CORS information and drop the depreciated predicates/types. It will also update the persisted query.

This PR also fixes the restored backup when a --upgrade is set to true in the export_backup command. Note that persisted query is not fixed in export_backup.

ee/backup/run.go

@@ -17,21 +17,23 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"math"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
-	"golang.org/x/sync/errgroup"
-
+	"github.com/dgraph-io/badger/v3"
 	"github.com/dgraph-io/badger/v3/options"
+	"golang.org/x/sync/errgroup"
 
 	"google.golang.org/grpc/credentials"
 
 	"github.com/dgraph-io/dgraph/ee"
 	"github.com/dgraph-io/dgraph/ee/enc"
 	"github.com/dgraph-io/dgraph/protos/pb"
+	"github.com/dgraph-io/dgraph/upgrade"
 	"github.com/dgraph-io/dgraph/worker"
 	"github.com/dgraph-io/dgraph/x"
 	"github.com/dgraph-io/ristretto/z"
@@ -59,6 +61,7 @@ var opt struct {
 	destination string
 	format      string
 	verbose     bool
+	upgrade     bool // used by export backup command.
 }
 
 func init() {
@@ -324,6 +327,7 @@ func initExportBackup() {
 		Annotations: map[string]string{"group": "tool"},
 	}
 
+	ExportBackup.Cmd.SetHelpTemplate(x.NonRootTemplate)
 	flag := ExportBackup.Cmd.Flags()
 	flag.StringVarP(&opt.location, "location", "l", "",
 		`Sets the location of the backup. Both file URIs and s3 are supported.
@@ -332,6 +336,10 @@ func initExportBackup() {
 		"The folder to which export the backups.")
 	flag.StringVarP(&opt.format, "format", "f", "rdf",
 		"The format of the export output. Accepts a value of either rdf or json")
+	flag.BoolVar(&opt.upgrade, "upgrade", false,
+		`If true, retrieve the CORS from DB and append at the end of GraphQL schema.
+		It also deletes the deprecated types and predicates.
+		Use this option when exporting a backup of 20.11 for loading onto 21.03.`)
 	enc.RegisterFlags(flag)
 }
 
@@ -376,6 +384,23 @@ func runExportBackup() error {
 				"inside DB at %s: %v", dir, err)
 			continue
 		}
+		if opt.upgrade && gid == 1 {
+			// Query the cors in badger db and append it at the end of GraphQL schema.
+			// This change was introduced in v21.03. Backups with 20.07 <= version < 21.03
+			// should apply this.
+			db, err := badger.OpenManaged(badger.DefaultOptions(dir).
+				WithNumVersionsToKeep(math.MaxInt32).
+				WithEncryptionKey(opt.key))
+			if err != nil {
+				return err
+			}
+			if err := upgrade.OfflineUpgradeFrom2011To2103(db); err != nil {
+				return errors.Wrapf(err, "while fixing cors")
+			}
+			if err := db.Close(); err != nil {
+				return err
+			}
+		}
 		eg.Go(func() error {
 			return worker.StoreExport(&pb.ExportRequest{
 				GroupId:     uint32(gid),

upgrade/change_list.go

@@ -52,5 +52,28 @@ func init() {
 				//	can't handle every scenario. So, it is best to let the user do it.
 			},
 		},
+		{
+			introducedIn: &version{major: 21, minor: 3, patch: 0},
+			changes: []*change{
+				{
+					name: "Upgrade Persistent Query",
+					description: "This updates the persisted query from old format to new format." +
+						"Persistent query had 2 predicates which have been merged into a single " +
+						"predicate dgraph.graphql.p_query. " +
+						"For more info, see: https://github.com/dgraph-io/dgraph/pull/7451",
+					minFromVersion: &version{major: 20, minor: 11, patch: 0},
+					applyFunc:      upgradePersitentQuery,
+				},
+				{
+					name: "Upgrade CORS",
+					description: "This updates GraphQL schema to contain the CORS information. " +
+						"Some of the dgraph internal predicates are removed in v21.03.0. " +
+						"dgraph.cors that used to store CORS information is one of them. " +
+						"For more info, see: https://github.com/dgraph-io/dgraph/pull/7431",
+					minFromVersion: &version{major: 20, minor: 11, patch: 0},
+					applyFunc:      upgradeCORS,
+				},
+			},
+		},
 	}
 }

upgrade/change_v20.03.0.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 
 	"github.com/dgraph-io/dgo/v200/protos/api"
+	"github.com/dgraph-io/dgraph/x"
 )
 
 const (
@@ -47,14 +48,11 @@ type rule struct {
 type rules []rule
 
 func upgradeACLRules() error {
-	dg, conn, err := getDgoClient(true)
-	if err != nil {
-		return fmt.Errorf("error getting dgo client: %w", err)
-	}
-	defer conn.Close()
+	dg, cb := x.GetDgraphClient(Upgrade.Conf, true)
+	defer cb()
 
 	data := make(map[string][]group)
-	if err = getQueryResult(dg, queryACLGroupsBefore_v20_03_0, &data); err != nil {
+	if err := getQueryResult(dg, queryACLGroupsBefore_v20_03_0, &data); err != nil {
 		return fmt.Errorf("error querying old ACL rules: %w", err)
 	}
 
@@ -118,7 +116,7 @@ func upgradeACLRules() error {
 
 	deleteOld := Upgrade.Conf.GetBool("deleteOld")
 	if deleteOld {
-		err = alterWithClient(dg, &api.Operation{
+		err := alterWithClient(dg, &api.Operation{
 			DropOp:    api.Operation_ATTR,
 			DropValue: "dgraph.group.acl",
 		})

upgrade/change_v20.07.0.go

@@ -22,6 +22,7 @@ import (
 	"github.com/dgraph-io/dgo/v200"
 	"github.com/dgraph-io/dgo/v200/protos/api"
 	"github.com/dgraph-io/dgraph/protos/pb"
+	"github.com/dgraph-io/dgraph/x"
 )
 
 const (
@@ -128,19 +129,16 @@ func upgradeAclTypeNames() error {
 	}
 
 	// get dgo client
-	dg, conn, err := getDgoClient(true)
-	if err != nil {
-		return fmt.Errorf("error getting dgo client: %w", err)
-	}
-	defer conn.Close()
+	dg, cb := x.GetDgraphClient(Upgrade.Conf, true)
+	defer cb()
 
 	// apply upgrades for old ACL type names, one by one.
 	for _, typeNameInfo := range aclTypeNameInfo {
-		if err = typeNameInfo.updateTypeName(dg); err != nil {
+		if err := typeNameInfo.updateTypeName(dg); err != nil {
 			return fmt.Errorf("error upgrading ACL type name from `%s` to `%s`: %w",
 				typeNameInfo.oldTypeName, typeNameInfo.newTypeName, err)
 		}
-		if err = typeNameInfo.updateTypeSchema(dg); err != nil {
+		if err := typeNameInfo.updateTypeSchema(dg); err != nil {
 			return fmt.Errorf("error upgrading schema for old ACL type `%s`: %w",
 				typeNameInfo.oldTypeName, err)
 		}