From c340278dfec68860c7c4d0fb03fc496952428679 Mon Sep 17 00:00:00 2001 From: Ping Yu Date: Mon, 19 Jun 2023 10:53:13 +0800 Subject: [PATCH] [to #346] Improve fault tolerance for TSO (#347) * improve fault tolerance for TSO Signed-off-by: Ping Yu * fix Signed-off-by: Ping Yu * fix gh Signed-off-by: Ping Yu * add LTS versions Signed-off-by: Ping Yu --------- Signed-off-by: Ping Yu --- .github/workflows/ci-br.yml | 25 ++++++++++++++++--------- br/pkg/backup/client.go | 15 +++++++++++---- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-br.yml b/.github/workflows/ci-br.yml index e8ce0464..2b677d48 100644 --- a/.github/workflows/ci-br.yml +++ b/.github/workflows/ci-br.yml @@ -10,6 +10,9 @@ on: permissions: contents: read +env: + GO_VERSION: 1.20.5 + jobs: br-check-tidy: runs-on: ubuntu-latest @@ -17,7 +20,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '1.18.1' + go-version: '${{ env.GO_VERSION }}' - name: make check/tidy shell: bash run: | @@ -29,7 +32,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '1.18.1' + go-version: '${{ env.GO_VERSION }}' - name: make check/golangci-lint shell: bash run: | @@ -41,7 +44,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '1.18.1' + go-version: '${{ env.GO_VERSION }}' - name: make check/gosec shell: bash run: | @@ -53,7 +56,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '1.18.1' + go-version: '${{ env.GO_VERSION }}' - name: make test_coverage shell: bash run: | @@ -65,7 +68,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - tikv_version: [v6.1.2, nightly] + tikv_version: [v6.1.6, v6.5.3, v7.1.0, nightly] api_version: [1, 2] with_tls: [false, true] include: @@ -79,7 +82,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-go@v2 with: - go-version: '1.18.1' + go-version: '${{ env.GO_VERSION }}' - name: install tiup run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh - name: download tikv binary @@ -107,11 +110,15 @@ jobs: echo "start tikv for apiversion ${{ matrix.with_tls }} and tls enabled" else echo -e "\napi-version = ${{ matrix.api_version }}\n" >> /home/runner/work/migration/migration/.github/config/br_rawkv.toml - /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/migration/migration/.github/config/br_rawkv.toml --pd.config /home/runner/work/migration/migration/.github/config/br_pd.toml &> raw.out 2>&1 & + /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --host 127.0.0.1 --kv 1 --without-monitor --kv.config /home/runner/work/migration/migration/.github/config/br_rawkv.toml --pd.port 2379 --pd.config /home/runner/work/migration/migration/.github/config/br_pd.toml &> raw.out 2>&1 & # The first run of `tiup` has to download all components so it'll take longer. - sleep 1m 30s + timeout 180 tail -f raw.out | grep -q 'PD Endpoints' + if [ $? -ne 0 ]; then + echo "Failed to start TiKV cluster" + exit 1 + fi # Parse PD address from `tiup` output - echo "PD_ADDR=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV + echo "PD_ADDR=127.0.0.1:2379" >> $GITHUB_ENV # Log the output echo "$(cat raw.out)" >&2 fi diff --git a/br/pkg/backup/client.go b/br/pkg/backup/client.go index 654ac55d..ff07cb85 100644 --- a/br/pkg/backup/client.go +++ b/br/pkg/backup/client.go @@ -101,11 +101,18 @@ func (bc *Client) GetTS(ctx context.Context, duration time.Duration, ts uint64) if ts > 0 { backupTS = ts } else { - p, l, err := bc.mgr.GetPDClient().GetTS(ctx) + var ( + physical int64 + logical int64 + ) + err = utils.WithRetry(ctx, func() error { + physical, logical, err = bc.mgr.GetPDClient().GetTS(ctx) + return errors.Trace(err) + }, utils.NewPDReqBackoffer()) if err != nil { return 0, errors.Trace(err) } - backupTS = oracle.ComposeTS(p, l) + backupTS = oracle.ComposeTS(physical, logical) switch { case duration < 0: @@ -115,10 +122,10 @@ func (bc *Client) GetTS(ctx context.Context, duration time.Duration, ts uint64) backupTime := oracle.GetTimeFromTS(backupTS) backupAgo := backupTime.Add(-duration) - if backupTS < oracle.ComposeTS(oracle.GetPhysical(backupAgo), l) { + if backupTS < oracle.ComposeTS(oracle.GetPhysical(backupAgo), logical) { return 0, errors.Annotate(berrors.ErrInvalidArgument, "backup ts overflow please choose a smaller timeago") } - backupTS = oracle.ComposeTS(oracle.GetPhysical(backupAgo), l) + backupTS = oracle.ComposeTS(oracle.GetPhysical(backupAgo), logical) } }