[close #220] TiKV-BR support key range (#221)

* support more args Signed-off-by: haojinming <[email protected]> * integration test Signed-off-by: haojinming <[email protected]> * remove useless code Signed-off-by: haojinming <[email protected]> * hide cipher args Signed-off-by: haojinming <[email protected]> * remove useless code Signed-off-by: haojinming <[email protected]> * go mod tidy Signed-off-by: haojinming <[email protected]> * remove useless code Signed-off-by: haojinming <[email protected]> * remove useless code Signed-off-by: haojinming <[email protected]> * address review comment Signed-off-by: haojinming <[email protected]> Signed-off-by: haojinming <[email protected]>
tikv · Sep 14, 2022 · 1640834 · 1640834
1 parent 088ea7b
commit 1640834
Show file tree

Hide file tree

Showing 19 changed files with 146 additions and 531 deletions.
diff --git a/br/go.mod b/br/go.mod
@@ -6,14 +6,12 @@ require (
 	cloud.google.com/go/storage v1.16.1
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0
-	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/aws/aws-sdk-go v1.35.3
 	github.com/cheggaaa/pb/v3 v3.0.8
 	github.com/cheynewallace/tabby v1.1.1
 	github.com/coreos/go-semver v0.3.0
 	github.com/docker/go-units v0.4.0
 	github.com/fsouza/fake-gcs-server v1.19.0
-	github.com/go-sql-driver/mysql v1.6.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/mock v1.6.0
 	github.com/google/btree v1.1.2
@@ -97,8 +95,3 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
-
-replace github.com/Sirupsen/logrus v1.5.0 => github.com/sirupsen/logrus v1.5.0
-
-// fix potential security issue(CVE-2020-26160) introduced by indirect dependency.
-replace github.com/dgrijalva/jwt-go => github.com/form3tech-oss/jwt-go v3.2.6-0.20210809144907-32ab6a8243d7+incompatible
diff --git a/br/go.sum b/br/go.sum
@@ -59,7 +59,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw=
 github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w=
-github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
 github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
@@ -165,6 +164,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
 github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
@@ -198,7 +198,6 @@ github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nI
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
-github.com/form3tech-oss/jwt-go v3.2.6-0.20210809144907-32ab6a8243d7+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -226,7 +225,6 @@ github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AE
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
-github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=

diff --git a/br/pkg/conn/conn.go b/br/pkg/conn/conn.go
@@ -8,7 +8,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"strings"
 	"sync"
@@ -26,6 +25,7 @@ import (
 	"github.com/tikv/client-go/v2/txnkv/txnlock"
 	berrors "github.com/tikv/migration/br/pkg/errors"
 	"github.com/tikv/migration/br/pkg/glue"
+	"github.com/tikv/migration/br/pkg/httputil"
 	"github.com/tikv/migration/br/pkg/logutil"
 	"github.com/tikv/migration/br/pkg/pdutil"
 	"github.com/tikv/migration/br/pkg/utils"
@@ -438,12 +438,9 @@ func GetTiKVApiVersion(ctx context.Context, pdClient pd.Client, tlsConf *tls.Con
 	} else if len(allStores) == 0 {
 		return kvrpcpb.APIVersion_V1, errors.New("store are empty")
 	}
+	httpClient := httputil.NewClient(tlsConf)
 	schema := "http"
-	httpClient := http.Client{}
 	if tlsConf != nil {
-		httpClient = http.Client{
-			Transport: &http.Transport{TLSClientConfig: tlsConf},
-		}
 		schema = "https"
 	}
 	url := fmt.Sprintf("%s://%s/config", schema, allStores[0].StatusAddress)

diff --git a/br/pkg/gluetikv/glue_test.go b/br/pkg/gluetikv/glue_test.go
@@ -15,12 +15,24 @@
 package gluetikv
 
 import (
+	"context"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
-func TestGetVersion(t *testing.T) {
+func TestGlue(t *testing.T) {
 	g := Glue{}
 	require.Regexp(t, "^BR(.|\n)*Release Version(.|\n)*Git Commit Hash(.|\n)*$", g.GetVersion())
+
+	require.True(t, g.OwnsStorage())
+
+	ctx := context.Background()
+	steps := int64(10)
+	updatCh := g.StartProgress(ctx, "test", steps, false)
+	for i := int64(0); i < steps; i++ {
+		updatCh.Inc()
+		g.Record("backup", 10)
+	}
+	updatCh.Close()
 }
diff --git a/br/pkg/restore/client.go b/br/pkg/restore/client.go
@@ -15,6 +15,7 @@ import (
 	"github.com/pingcap/kvproto/pkg/kvrpcpb"
 	"github.com/pingcap/log"
 	"github.com/tikv/client-go/v2/oracle"
+	"github.com/tikv/client-go/v2/util/codec"
 	"github.com/tikv/migration/br/pkg/conn"
 	berrors "github.com/tikv/migration/br/pkg/errors"
 	"github.com/tikv/migration/br/pkg/glue"
@@ -256,6 +257,14 @@ func (rc *Client) RestoreRaw(
 	errCh := make(chan error, len(files))
 	eg, ectx := errgroup.WithContext(ctx)
 	defer close(errCh)
+	if rc.dstAPIVersion == kvrpcpb.APIVersion_V2 {
+		startKey = codec.EncodeBytes(nil, startKey)
+		endKey = codec.EncodeBytes(nil, endKey)
+		for _, file := range files {
+			file.StartKey = codec.EncodeBytes(nil, file.StartKey)
+			file.EndKey = codec.EncodeBytes(nil, file.EndKey)
+		}
+	}
 
 	err := rc.fileImporter.SetRawRange(startKey, endKey)
 	if err != nil {

diff --git a/br/pkg/restore/util.go b/br/pkg/restore/util.go
@@ -7,7 +7,6 @@ import (
 	"context"
 	"strings"
 
-	_ "github.com/go-sql-driver/mysql" // mysql driver
 	"github.com/pingcap/errors"
 	backuppb "github.com/pingcap/kvproto/pkg/brpb"
 	"github.com/pingcap/kvproto/pkg/import_sstpb"

diff --git a/br/pkg/storage/s3.go b/br/pkg/storage/s3.go
@@ -180,8 +180,6 @@ func defineS3Flags(flags *pflag.FlagSet) {
 		"Leave empty to use S3 owned key.")
 	flags.String(s3ACLOption, "", "(experimental) Set the S3 canned ACLs, e.g. authenticated-read")
 	flags.String(s3ProviderOption, "", "(experimental) Set the S3 provider, e.g. aws, alibaba, ceph")
-	_ = flags.MarkHidden(s3EndpointOption)
-	_ = flags.MarkHidden(s3RegionOption)
 	_ = flags.MarkHidden(s3StorageClassOption)
 	_ = flags.MarkHidden(s3SseOption)
 	_ = flags.MarkHidden(s3SseKmsKeyIDOption)

diff --git a/br/pkg/task/backup_raw.go b/br/pkg/task/backup_raw.go
@@ -61,9 +61,6 @@ func DefineRawBackupFlags(command *cobra.Command) {
 	// This flag can impact the online cluster, so hide it in case of abuse.
 	_ = command.Flags().MarkHidden(flagCompressionType)
 	_ = command.Flags().MarkHidden(flagRemoveSchedulers)
-	_ = command.Flags().MarkHidden(flagStartKey)
-	_ = command.Flags().MarkHidden(flagEndKey)
-	_ = command.Flags().MarkHidden(flagKeyFormat)
 }
 
 // CalcChecksumFromBackupMeta read the backup meta and return Checksum

diff --git a/br/pkg/task/common.go b/br/pkg/task/common.go
@@ -89,13 +89,9 @@ func DefineCommonFlags(flags *pflag.FlagSet) {
 	flags.String(flagKey, "", "Private key path for TLS connection")
 	flags.Uint(flagChecksumConcurrency, defaultChecksumConcurrency, "The concurrency of table checksumming")
 	_ = flags.MarkHidden(flagSendCreds)
-	_ = flags.MarkHidden(flagCA)
-	_ = flags.MarkHidden(flagCert)
-	_ = flags.MarkHidden(flagKey)
 	_ = flags.MarkHidden(flagChecksumConcurrency)
 
 	flags.Uint64(flagRateLimit, unlimited, "The rate limit of the task, MB/s per node")
-	_ = flags.MarkHidden(flagRateLimit)
 	flags.Bool(flagChecksum, false, "Run checksum at end of task")
 	// Default concurrency is different for backup and restore.
 	// Leave it 0 and let them adjust the value.

diff --git a/br/pkg/task/restore_raw.go b/br/pkg/task/restore_raw.go
@@ -24,9 +24,6 @@ func DefineRawRestoreFlags(command *cobra.Command) {
 	command.Flags().StringP(flagKeyFormat, "", "hex", "start/end key format, support raw|escaped|hex")
 	command.Flags().StringP(flagStartKey, "", "", "restore raw kv start key, key is inclusive")
 	command.Flags().StringP(flagEndKey, "", "", "restore raw kv end key, key is exclusive")
-	_ = command.Flags().MarkHidden(flagKeyFormat)
-	_ = command.Flags().MarkHidden(flagStartKey)
-	_ = command.Flags().MarkHidden(flagEndKey)
 	DefineRestoreCommonFlags(command.PersistentFlags())
 }
 
@@ -130,13 +127,6 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR
 			End:   file.EndKey,
 		})
 	}
-	if needEncodeKey {
-		for _, file := range files {
-			keyRange := utils.EncodeKeyRange(file.StartKey, file.EndKey)
-			file.StartKey = keyRange.Start
-			file.EndKey = keyRange.End
-		}
-	}
 
 	err = client.RestoreRaw(ctx, cfg.StartKey, cfg.EndKey, files, updateCh)
 	if err != nil {

diff --git a/br/pkg/utils/backoff.go b/br/pkg/utils/backoff.go
@@ -4,7 +4,6 @@ package utils
 
 import (
 	"context"
-	"database/sql"
 	"io"
 	"time"
 
@@ -113,7 +112,7 @@ func (bo *pdReqBackoffer) NextBackoff(err error) time.Duration {
 	// bo.attempt--
 	e := errors.Cause(err)
 	switch e { // nolint:errorlint
-	case nil, context.Canceled, context.DeadlineExceeded, io.EOF, sql.ErrNoRows:
+	case nil, context.Canceled, context.DeadlineExceeded, io.EOF:
 		// Excepted error, finish the operation
 		bo.delayTime = 0
 		bo.attempt = 0

diff --git a/br/pkg/utils/db.go b/br/pkg/utils/db.go
diff --git a/br/pkg/utils/retry.go b/br/pkg/utils/retry.go
@@ -4,7 +4,6 @@ package utils
 
 import (
 	"context"
-	"database/sql"
 	stderrors "errors"
 	"io"
 	"net"
@@ -105,7 +104,7 @@ func isSingleRetryableError(err error) bool {
 	err = errors.Cause(err)
 
 	switch err {
-	case nil, context.Canceled, context.DeadlineExceeded, io.EOF, sql.ErrNoRows:
+	case nil, context.Canceled, context.DeadlineExceeded, io.EOF:
 		return false
 	}
 

diff --git a/br/pkg/version/build/info.go b/br/pkg/version/build/info.go
@@ -26,8 +26,6 @@ type AppName string
 var (
 	// BR is the name of BR binary.
 	BR AppName = "Backup & Restore (BR)"
-	// Lightning is the name of Lightning binary.
-	Lightning AppName = "TiDB-Lightning"
 )
 
 // LogInfo logs version information.

diff --git a/br/pkg/version/build/info_test.go b/br/pkg/version/build/info_test.go
@@ -21,5 +21,4 @@ func TestInfo(t *testing.T) {
 
 func TestLogInfo(t *testing.T) {
 	LogInfo(BR)
-	LogInfo(Lightning)
 }