[to #346] Improve fault tolerance for TSO (#347)

* improve fault tolerance for TSO Signed-off-by: Ping Yu <[email protected]> * fix Signed-off-by: Ping Yu <[email protected]> * fix gh Signed-off-by: Ping Yu <[email protected]> * add LTS versions Signed-off-by: Ping Yu <[email protected]> --------- Signed-off-by: Ping Yu <[email protected]>
tikv · Jun 19, 2023 · c340278 · c340278
1 parent 37f5972
commit c340278
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 13 deletions.
diff --git a/.github/workflows/ci-br.yml b/.github/workflows/ci-br.yml
@@ -10,14 +10,17 @@ on:
 permissions:
   contents: read
 
+env:
+  GO_VERSION: 1.20.5
+
 jobs:
   br-check-tidy:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.18.1'
+          go-version: '${{ env.GO_VERSION }}'
       - name: make check/tidy
         shell: bash
         run: |
@@ -29,7 +32,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.18.1'
+          go-version: '${{ env.GO_VERSION }}'
       - name: make check/golangci-lint
         shell: bash
         run: |
@@ -41,7 +44,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.18.1'
+          go-version: '${{ env.GO_VERSION }}'
       - name: make check/gosec
         shell: bash
         run: |
@@ -53,7 +56,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.18.1'
+          go-version: '${{ env.GO_VERSION }}'
       - name: make test_coverage
         shell: bash
         run: |
@@ -65,7 +68,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        tikv_version: [v6.1.2, nightly]
+        tikv_version: [v6.1.6, v6.5.3, v7.1.0, nightly]
         api_version: [1, 2]
         with_tls: [false, true]
         include:
@@ -79,7 +82,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.18.1'
+          go-version: '${{ env.GO_VERSION }}'
       - name: install tiup
         run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
       - name: download tikv binary
@@ -107,11 +110,15 @@ jobs:
             echo "start tikv for apiversion ${{ matrix.with_tls }} and tls enabled"
           else
             echo -e "\napi-version = ${{ matrix.api_version }}\n" >> /home/runner/work/migration/migration/.github/config/br_rawkv.toml
-            /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/migration/migration/.github/config/br_rawkv.toml --pd.config /home/runner/work/migration/migration/.github/config/br_pd.toml &> raw.out 2>&1 &
+            /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --host 127.0.0.1 --kv 1 --without-monitor --kv.config /home/runner/work/migration/migration/.github/config/br_rawkv.toml --pd.port 2379 --pd.config /home/runner/work/migration/migration/.github/config/br_pd.toml &> raw.out 2>&1 &
             # The first run of `tiup` has to download all components so it'll take longer.
-            sleep 1m 30s
+            timeout 180 tail -f raw.out | grep -q 'PD Endpoints'
+            if [ $? -ne 0 ]; then
+              echo "Failed to start TiKV cluster"
+              exit 1
+            fi
             # Parse PD address from `tiup` output
-            echo "PD_ADDR=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV
+            echo "PD_ADDR=127.0.0.1:2379" >> $GITHUB_ENV
             # Log the output
             echo "$(cat raw.out)" >&2
           fi

diff --git a/br/pkg/backup/client.go b/br/pkg/backup/client.go
@@ -101,11 +101,18 @@ func (bc *Client) GetTS(ctx context.Context, duration time.Duration, ts uint64)
 	if ts > 0 {
 		backupTS = ts
 	} else {
-		p, l, err := bc.mgr.GetPDClient().GetTS(ctx)
+		var (
+			physical int64
+			logical  int64
+		)
+		err = utils.WithRetry(ctx, func() error {
+			physical, logical, err = bc.mgr.GetPDClient().GetTS(ctx)
+			return errors.Trace(err)
+		}, utils.NewPDReqBackoffer())
 		if err != nil {
 			return 0, errors.Trace(err)
 		}
-		backupTS = oracle.ComposeTS(p, l)
+		backupTS = oracle.ComposeTS(physical, logical)
 
 		switch {
 		case duration < 0:
@@ -115,10 +122,10 @@ func (bc *Client) GetTS(ctx context.Context, duration time.Duration, ts uint64)
 
 			backupTime := oracle.GetTimeFromTS(backupTS)
 			backupAgo := backupTime.Add(-duration)
-			if backupTS < oracle.ComposeTS(oracle.GetPhysical(backupAgo), l) {
+			if backupTS < oracle.ComposeTS(oracle.GetPhysical(backupAgo), logical) {
 				return 0, errors.Annotate(berrors.ErrInvalidArgument, "backup ts overflow please choose a smaller timeago")
 			}
-			backupTS = oracle.ComposeTS(oracle.GetPhysical(backupAgo), l)
+			backupTS = oracle.ComposeTS(oracle.GetPhysical(backupAgo), logical)
 		}
 	}