Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[close #220] TiKV-BR support key range #221

Merged
merged 10 commits into from
Sep 14, 2022
Merged

[close #220] TiKV-BR support key range #221

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
9f23b9b
support more args
haojinming Aug 19, 2022
295ce8e
integration test
haojinming Aug 22, 2022
d4e2a0d
remove useless code
haojinming Sep 7, 2022
99e9e2c
hide cipher args
haojinming Sep 7, 2022
7b7c38f
remove useless code
haojinming Sep 7, 2022
a519b23
go mod tidy
haojinming Sep 7, 2022
f959a8d
remove useless code
haojinming Sep 7, 2022
268b166
remove useless code
haojinming Sep 7, 2022
bed055d
address review comment
haojinming Sep 13, 2022
99f9f02
Merge branch 'main' into br_support_more_args
haojinming Sep 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions br/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,12 @@ require (
cloud.google.com/go/storage v1.16.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.12.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/aws/aws-sdk-go v1.35.3
github.com/cheggaaa/pb/v3 v3.0.8
github.com/cheynewallace/tabby v1.1.1
github.com/coreos/go-semver v0.3.0
github.com/docker/go-units v0.4.0
github.com/fsouza/fake-gcs-server v1.19.0
github.com/go-sql-driver/mysql v1.6.0
github.com/gogo/protobuf v1.3.2
github.com/golang/mock v1.6.0
github.com/google/btree v1.1.2
Expand Down Expand Up @@ -97,8 +95,3 @@ require (
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

replace github.com/Sirupsen/logrus v1.5.0 => github.com/sirupsen/logrus v1.5.0

// fix potential security issue(CVE-2020-26160) introduced by indirect dependency.
replace github.com/dgrijalva/jwt-go => github.com/form3tech-oss/jwt-go v3.2.6-0.20210809144907-32ab6a8243d7+incompatible
4 changes: 1 addition & 3 deletions br/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw=
github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w=
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
Expand Down Expand Up @@ -165,6 +164,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
Expand Down Expand Up @@ -198,7 +198,6 @@ github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nI
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/form3tech-oss/jwt-go v3.2.6-0.20210809144907-32ab6a8243d7+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
Expand Down Expand Up @@ -226,7 +225,6 @@ github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AE
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
Expand Down
7 changes: 2 additions & 5 deletions br/pkg/conn/conn.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"os"
"strings"
"sync"
Expand All @@ -26,6 +25,7 @@ import (
"github.com/tikv/client-go/v2/txnkv/txnlock"
berrors "github.com/tikv/migration/br/pkg/errors"
"github.com/tikv/migration/br/pkg/glue"
"github.com/tikv/migration/br/pkg/httputil"
"github.com/tikv/migration/br/pkg/logutil"
"github.com/tikv/migration/br/pkg/pdutil"
"github.com/tikv/migration/br/pkg/utils"
Expand Down Expand Up @@ -438,12 +438,9 @@ func GetTiKVApiVersion(ctx context.Context, pdClient pd.Client, tlsConf *tls.Con
} else if len(allStores) == 0 {
return kvrpcpb.APIVersion_V1, errors.New("store are empty")
}
httpClient := httputil.NewClient(tlsConf)
schema := "http"
httpClient := http.Client{}
if tlsConf != nil {
httpClient = http.Client{
Transport: &http.Transport{TLSClientConfig: tlsConf},
}
schema = "https"
}
url := fmt.Sprintf("%s://%s/config", schema, allStores[0].StatusAddress)
Expand Down
14 changes: 13 additions & 1 deletion br/pkg/gluetikv/glue_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,24 @@
package gluetikv

import (
"context"
"testing"

"github.com/stretchr/testify/require"
)

func TestGetVersion(t *testing.T) {
func TestGlue(t *testing.T) {
g := Glue{}
require.Regexp(t, "^BR(.|\n)*Release Version(.|\n)*Git Commit Hash(.|\n)*$", g.GetVersion())

require.True(t, g.OwnsStorage())

ctx := context.Background()
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is verified in this block ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just to cover the code to verify no crash occurs.

steps := int64(10)
updatCh := g.StartProgress(ctx, "test", steps, false)
for i := int64(0); i < steps; i++ {
updatCh.Inc()
g.Record("backup", 10)
}
updatCh.Close()
}
9 changes: 9 additions & 0 deletions br/pkg/restore/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/pingcap/kvproto/pkg/kvrpcpb"
"github.com/pingcap/log"
"github.com/tikv/client-go/v2/oracle"
"github.com/tikv/client-go/v2/util/codec"
"github.com/tikv/migration/br/pkg/conn"
berrors "github.com/tikv/migration/br/pkg/errors"
"github.com/tikv/migration/br/pkg/glue"
Expand Down Expand Up @@ -256,6 +257,14 @@ func (rc *Client) RestoreRaw(
errCh := make(chan error, len(files))
eg, ectx := errgroup.WithContext(ctx)
defer close(errCh)
if rc.dstAPIVersion == kvrpcpb.APIVersion_V2 {
startKey = codec.EncodeBytes(nil, startKey)
endKey = codec.EncodeBytes(nil, endKey)
for _, file := range files {
file.StartKey = codec.EncodeBytes(nil, file.StartKey)
file.EndKey = codec.EncodeBytes(nil, file.EndKey)
}
}

err := rc.fileImporter.SetRawRange(startKey, endKey)
if err != nil {
Expand Down
1 change: 0 additions & 1 deletion br/pkg/restore/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"context"
"strings"

_ "github.com/go-sql-driver/mysql" // mysql driver
"github.com/pingcap/errors"
backuppb "github.com/pingcap/kvproto/pkg/brpb"
"github.com/pingcap/kvproto/pkg/import_sstpb"
Expand Down
2 changes: 0 additions & 2 deletions br/pkg/storage/s3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,8 +180,6 @@ func defineS3Flags(flags *pflag.FlagSet) {
"Leave empty to use S3 owned key.")
flags.String(s3ACLOption, "", "(experimental) Set the S3 canned ACLs, e.g. authenticated-read")
flags.String(s3ProviderOption, "", "(experimental) Set the S3 provider, e.g. aws, alibaba, ceph")
_ = flags.MarkHidden(s3EndpointOption)
_ = flags.MarkHidden(s3RegionOption)
_ = flags.MarkHidden(s3StorageClassOption)
_ = flags.MarkHidden(s3SseOption)
_ = flags.MarkHidden(s3SseKmsKeyIDOption)
Expand Down
3 changes: 0 additions & 3 deletions br/pkg/task/backup_raw.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,6 @@ func DefineRawBackupFlags(command *cobra.Command) {
// This flag can impact the online cluster, so hide it in case of abuse.
_ = command.Flags().MarkHidden(flagCompressionType)
_ = command.Flags().MarkHidden(flagRemoveSchedulers)
_ = command.Flags().MarkHidden(flagStartKey)
_ = command.Flags().MarkHidden(flagEndKey)
_ = command.Flags().MarkHidden(flagKeyFormat)
}

// CalcChecksumFromBackupMeta read the backup meta and return Checksum
Expand Down
4 changes: 0 additions & 4 deletions br/pkg/task/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,13 +89,9 @@ func DefineCommonFlags(flags *pflag.FlagSet) {
flags.String(flagKey, "", "Private key path for TLS connection")
flags.Uint(flagChecksumConcurrency, defaultChecksumConcurrency, "The concurrency of table checksumming")
_ = flags.MarkHidden(flagSendCreds)
_ = flags.MarkHidden(flagCA)
_ = flags.MarkHidden(flagCert)
_ = flags.MarkHidden(flagKey)
_ = flags.MarkHidden(flagChecksumConcurrency)

flags.Uint64(flagRateLimit, unlimited, "The rate limit of the task, MB/s per node")
_ = flags.MarkHidden(flagRateLimit)
flags.Bool(flagChecksum, false, "Run checksum at end of task")
// Default concurrency is different for backup and restore.
// Leave it 0 and let them adjust the value.
Expand Down
10 changes: 0 additions & 10 deletions br/pkg/task/restore_raw.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,6 @@ func DefineRawRestoreFlags(command *cobra.Command) {
command.Flags().StringP(flagKeyFormat, "", "hex", "start/end key format, support raw|escaped|hex")
command.Flags().StringP(flagStartKey, "", "", "restore raw kv start key, key is inclusive")
command.Flags().StringP(flagEndKey, "", "", "restore raw kv end key, key is exclusive")
_ = command.Flags().MarkHidden(flagKeyFormat)
_ = command.Flags().MarkHidden(flagStartKey)
_ = command.Flags().MarkHidden(flagEndKey)
DefineRestoreCommonFlags(command.PersistentFlags())
}

Expand Down Expand Up @@ -130,13 +127,6 @@ func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreR
End: file.EndKey,
})
}
if needEncodeKey {
for _, file := range files {
keyRange := utils.EncodeKeyRange(file.StartKey, file.EndKey)
file.StartKey = keyRange.Start
file.EndKey = keyRange.End
}
}

err = client.RestoreRaw(ctx, cfg.StartKey, cfg.EndKey, files, updateCh)
if err != nil {
Expand Down
3 changes: 1 addition & 2 deletions br/pkg/utils/backoff.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ package utils

import (
"context"
"database/sql"
"io"
"time"

Expand Down Expand Up @@ -113,7 +112,7 @@ func (bo *pdReqBackoffer) NextBackoff(err error) time.Duration {
// bo.attempt--
e := errors.Cause(err)
switch e { // nolint:errorlint
case nil, context.Canceled, context.DeadlineExceeded, io.EOF, sql.ErrNoRows:
case nil, context.Canceled, context.DeadlineExceeded, io.EOF:
// Excepted error, finish the operation
bo.delayTime = 0
bo.attempt = 0
Expand Down
32 changes: 0 additions & 32 deletions br/pkg/utils/db.go
View file
Open in desktop

This file was deleted.

3 changes: 1 addition & 2 deletions br/pkg/utils/retry.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ package utils

import (
"context"
"database/sql"
stderrors "errors"
"io"
"net"
Expand Down Expand Up @@ -105,7 +104,7 @@ func isSingleRetryableError(err error) bool {
err = errors.Cause(err)

switch err {
case nil, context.Canceled, context.DeadlineExceeded, io.EOF, sql.ErrNoRows:
case nil, context.Canceled, context.DeadlineExceeded, io.EOF:
return false
}

Expand Down
2 changes: 0 additions & 2 deletions br/pkg/version/build/info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ type AppName string
var (
// BR is the name of BR binary.
BR AppName = "Backup & Restore (BR)"
// Lightning is the name of Lightning binary.
Lightning AppName = "TiDB-Lightning"
)

// LogInfo logs version information.
Expand Down
1 change: 0 additions & 1 deletion br/pkg/version/build/info_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,4 @@ func TestInfo(t *testing.T) {

func TestLogInfo(t *testing.T) {
LogInfo(BR)
LogInfo(Lightning)
}
Loading