Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[close #229] Add integration test coverage report for TiKV-BR #230

Merged
merged 7 commits into from
Sep 20, 2022
Merged

[close #229] Add integration test coverage report for TiKV-BR #230

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c672def
add integration test
haojinming Aug 31, 2022
0db6c24
fix coverage report
haojinming Sep 19, 2022
6d228d8
address review comment: use tmp file
haojinming Sep 19, 2022
9c4d2d5
codecov upload multi files
haojinming Sep 19, 2022
87a1685
codecov upload multi files
haojinming Sep 19, 2022
c9e1158
codecov upload multi files
haojinming Sep 19, 2022
d132c49
remove useless code
haojinming Sep 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 3 additions & 31 deletions .github/workflows/ci-br.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,8 @@ jobs:
shell: bash
run: |
cd br
make test_coverage
make test
make report_coverage
br-integration-test:
name: br-integration-test-${{ matrix.tikv_version }}-apiv${{ matrix.api_version }}
runs-on: ubuntu-latest
Expand Down Expand Up @@ -90,36 +91,7 @@ jobs:
run: |
cd br
make test/integration
make report_coverage
- name: display backup log
run: |
cat /tmp/br.log*
# comments low version CI as they fail without checksum & split region capability.
# Plan to support low version with feature gate in the future.
# br-integration-test-5X:
# # Separate integration tests of v5.X, to distinguish whether failure would be caused by compatibility of old versions.
# name: br-integration-test-5X-${{ matrix.tikv_version }}
# runs-on: ubuntu-latest
# strategy:
# matrix:
# tikv_version: [v5.0.0, v5.1.0, v5.2.0, v5.3.0, v5.4.0, v6.0.0]
# steps:
# - uses: actions/checkout@v2
# - uses: actions/setup-go@v2
# with:
# go-version: '1.18.1'
# - name: install tiup
# run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
# - name: start tikv cluster
# run: |
# # start tikv in apiv1ttl
# /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/migration/migration/.github/config/br_rawkv.toml --pd.config /home/runner/work/migration/migration/.github/config/br_pd.toml &> raw.out 2>&1 &
# # The first run of `tiup` has to download all components so it'll take longer.
# sleep 1m 30s
# # Parse PD address from `tiup` output
# echo "PD_ADDR=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV
# # Log the output
# echo "$(cat raw.out)" >&2
# - name: run integration test
# run: |
# cd br
# make test/integration
19 changes: 7 additions & 12 deletions br/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,22 +52,23 @@ check/gosec:
$(GO) install github.com/securego/gosec/v2/cmd/[email protected]
gosec -fmt=junit-xml -out=results.xml -stdout -verbose=text -exclude=G103,G104,G204,G304,G307,G401,G404,G501,G505,G601 ./...

test: tools/bin/gocov tools/bin/gocov-xml
test:
make failpoint/enable
export TZ='Asia/Shanghai'
mkdir -p $(COVERAGE_DIR)
$(GO) test -p $(TEST_PARALLEL) -race -ldflags '$(LDFLAGS)' -tags leak $$($(PACKAGES)) -coverprofile=$(COVERAGE_DIR)/coverage.raw || ( make failpoint/disable && exit 1 )
tools/bin/gocov convert $(COVERAGE_DIR)/coverage.raw | tools/bin/gocov-xml > $(COVERAGE_DIR)/coverage.xml
$(GO) test -p $(TEST_PARALLEL) -race -ldflags '$(LDFLAGS)' -tags leak $$($(PACKAGES)) -coverprofile=$(COVERAGE_DIR)/cov.unit.out || ( make failpoint/disable && exit 1 )
make failpoint/disable

test_coverage: test
@bash <(curl -s https://codecov.io/bash) -F br -f $(COVERAGE_DIR)/coverage.raw -t $(TIKV_MIGRATION_CODECOV_TOKEN)
report_coverage:
@bash <(curl -s https://codecov.io/bash) -F br -f $(COVERAGE_DIR)/cov.* -t $(TIKV_MIGRATION_CODECOV_TOKEN)

test/integration: build/br-test build/rawkv-integration-test
mkdir -p $(COVERAGE_DIR)
./bin/rawkv_test --pd=${PD_ADDR} \
--br='${TEST_BIN_PATH}' \
--br-storage=${BR_LOCAL_STORE} \
--api-version=${API_VERSION}
--api-version=${API_VERSION} \
--coverage=$(COVERAGE_DIR)/cov.integration.out

failpoint/enable: tools/bin/failpoint-ctl
find `pwd` -type d | grep -vE "(\.git|tools)" | xargs tools/bin/failpoint-ctl enable
Expand All @@ -78,12 +79,6 @@ failpoint/disable: tools/bin/failpoint-ctl
tools/bin/golangci-lint:
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./tools/bin v1.45.1

tools/bin/gocov: tools/check/go.mod
cd tools/check && $(GO) build -mod=mod -o ../bin/gocov github.com/axw/gocov/gocov

tools/bin/gocov-xml: tools/check/go.mod
cd tools/check && $(GO) build -mod=mod -o ../bin/gocov-xml github.com/AlekSi/gocov-xml

tools/bin/failpoint-ctl: tools/check/go.mod
cd tools/check && $(GO) build -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl

Expand Down
24 changes: 17 additions & 7 deletions br/tests/rawkv/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,18 @@ var (
apiVersionInt = flag.Uint("api-version", 1, "Api version of tikv-server")
br = flag.String("br", "br", "The br binary to be tested.")
brStorage = flag.String("br-storage", "local:///tmp/backup_restore_test", "The url to store SST files of backup/resotre.")
coverageFile = flag.String("coverage", "coverage.out", "The coverage profile of test.")
)

type RawKVBRTester struct {
name string
pdAddr string
apiVersion kvrpcpb.APIVersion
br string
brStorage string
rawkvClient *rawkv.Client
pdClient pd.Client
runCnt int // runCnt is used to distinguish `backup` and `restore` commands.
}

func NewPDClient(ctx context.Context, pdAddrs string) (pd.Client, error) {
Expand All @@ -60,7 +63,7 @@ func NewPDClient(ctx context.Context, pdAddrs string) (pd.Client, error) {
)
}

func NewRawKVBRTester(ctx context.Context, pd, br, storage string, version kvrpcpb.APIVersion) (*RawKVBRTester, error) {
func NewRawKVBRTester(ctx context.Context, name, pd, br, storage string, version kvrpcpb.APIVersion) (*RawKVBRTester, error) {
cli, err := rawkv.NewClientWithOpts(context.TODO(), []string{pd},
rawkv.WithAPIVersion(version))
if err != nil {
Expand All @@ -73,12 +76,14 @@ func NewRawKVBRTester(ctx context.Context, pd, br, storage string, version kvrpc
return nil, err
}
return &RawKVBRTester{
name: name,
pdAddr: pd,
br: br,
apiVersion: version,
brStorage: storage,
rawkvClient: cli,
pdClient: pdClient,
runCnt: 0,
}, nil
}

Expand Down Expand Up @@ -201,7 +206,11 @@ func (t *RawKVBRTester) InjectFailpoint(failpoint string) error {

func (t *RawKVBRTester) ExecBRCmd(ctx context.Context, cmdStr string) ([]byte, error) {
log.Info("exec br cmd", zap.String("br", t.br), zap.String("args", cmdStr))
cmd := exec.CommandContext(ctx, t.br, strings.Split(cmdStr, " ")...)
cmdParameter := []string{fmt.Sprintf("-test.coverprofile=%s_%s_%d", *coverageFile, t.name, t.runCnt)}
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about just create a unique temporary file (see os.CreateTemp) for the coverage ? Then we don't need the case name & run count.

(If use temporary file, it may need to clear the coverage output path before running test)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks better. I have changed to temp file for coverage file. PTAL~

//cmdParameter := []string{}
haojinming marked this conversation as resolved.
Show resolved Hide resolved
cmdParameter = append(cmdParameter, strings.Split(cmdStr, " ")...)
cmd := exec.CommandContext(ctx, t.br, cmdParameter...)
t.runCnt += 1 // runCnt is used to distinguish `backup` and `restore` commands.
return cmd.Output()
}

Expand Down Expand Up @@ -292,7 +301,7 @@ func runBackupAndRestore(ctx context.Context, tester *RawKVBRTester, prefix, sta
log.Info("Checksum pass")
}

func runTestWithFailPoint(failpoint string, prefix []byte, backupRange *kvrpcpb.KeyRange) {
func runTestWithFailPoint(caseName, failpoint string, prefix []byte, backupRange *kvrpcpb.KeyRange) {
apiVersion := kvrpcpb.APIVersion_V1TTL
if *apiVersionInt == 2 {
apiVersion = kvrpcpb.APIVersion_V2
Expand All @@ -301,7 +310,7 @@ func runTestWithFailPoint(failpoint string, prefix []byte, backupRange *kvrpcpb.

fmt.Println("test api version", apiVersion)

tester, err := NewRawKVBRTester(ctx, *pdAddr, *br, *brStorage, apiVersion)
tester, err := NewRawKVBRTester(ctx, caseName, *pdAddr, *br, *brStorage, apiVersion)
if err != nil {
log.Panic("New Tester Fail", zap.Error(err))
}
Expand Down Expand Up @@ -351,9 +360,10 @@ func main() {
{StartKey: []byte{}, EndKey: []byte{}},
{StartKey: q1Key, EndKey: q3Key},
}
for _, failpoint := range failpoints {
for _, backupRange := range backupRanges {
runTestWithFailPoint(failpoint, prefix, &backupRange)
for i, failpoint := range failpoints {
for j, backupRange := range backupRanges {
caseName := fmt.Sprintf("case_%d_%d", i, j)
runTestWithFailPoint(caseName, failpoint, prefix, &backupRange)
}
}
}