From bfd6e2221983536f0a24fc22a0da829ac5af15e3 Mon Sep 17 00:00:00 2001 From: Joel Hendrix Date: Wed, 22 Jul 2020 12:04:52 -0700 Subject: [PATCH] Build track 2 content in module-aware mode --- azure-pipelines.yml | 4 +- eng/pipelines/templates/steps/build-test.yml | 61 +++++++++++++++----- eng/pipelines/templates/steps/set-scope.yml | 2 +- eng/scripts/get_module_dirs.ps1 | 15 +++++ eng/scripts/get_test_dirs.ps1 | 17 ++++++ eng/scripts/scoped_discover.ps1 | 2 +- 6 files changed, 84 insertions(+), 17 deletions(-) create mode 100644 eng/scripts/get_module_dirs.ps1 create mode 100644 eng/scripts/get_test_dirs.ps1 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7be5c88b843b..4ff4639d5dba 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,7 +26,7 @@ jobs: GOPATH: '$(system.defaultWorkingDirectory)/work' sdkPath: '$(GOPATH)/src/github.com/$(build.repository.name)' IGNORE_BREAKING_CHANGES: true - go.list.filter: '| grep -v vendor' + go.list.filter: '| grep -v vendor | grep -v azure-sdk-for-go/sdk' steps: - task: GoTool@0 @@ -57,7 +57,7 @@ jobs: - script: go build -v $(go list ./... $(go.list.filter)) workingDirectory: '$(sdkPath)' displayName: 'Build' - - script: go test $(dirname $(find . -path ./vendor -prune -o -path ./sdk -prune -o -name '*_test.go' -print) | sort -u) + - script: go test $(dirname $(find . -path ./vendor -prune -o -path ./sdk -prune -o -path ./sdk -prune -o -name '*_test.go' -print) | sort -u) workingDirectory: '$(sdkPath)' displayName: 'Run Tests' - script: go run ./tools/apidiff/main.go packages ./services FETCH_HEAD~1 FETCH_HEAD --copyrepo --breakingchanges || $IGNORE_BREAKING_CHANGES diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index 3570fccabf7a..536460220177 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -12,43 +12,78 @@ steps: go get github.com/axw/gocov/gocov go get github.com/AlekSi/gocov-xml go get github.com/matm/gocov-html + go get -u github.com/wadey/gocovmerge displayName: "Install Coverage and Junit Dependencies" workingDirectory: '${{parameters.GoWorkspace}}' - pwsh: | - go vet $(go list $(SCOPE)) - displayName: 'Vet' + $modDirs = (./eng/scripts/get_module_dirs.ps1 -serviceDir $(SCOPE)) + foreach ($md in $modDirs) { + pushd $md + Write-Host "##[command]Executing go build -v $md" + go build -v + } + displayName: 'Build' workingDirectory: '${{parameters.GoWorkspace}}' + env: + GO111MODULE: 'on' - pwsh: | - go build -v $(go list $(SCOPE)) - displayName: 'Build' + $modDirs = (./eng/scripts/get_module_dirs.ps1 -serviceDir $(SCOPE)) + foreach ($md in $modDirs) { + pushd $md + Write-Host "##[command]Executing go vet $md" + go vet + } + displayName: 'Vet' workingDirectory: '${{parameters.GoWorkspace}}' + env: + GO111MODULE: 'on' - pwsh: | - go test -race -v -coverprofile coverage.txt -covermode atomic $(SCOPE) | go-junit-report > $(Build.SourcesDirectory)/report.xml + $testDirs = (./eng/scripts/get_test_dirs.ps1 -serviceDir $(SCOPE)) + foreach ($td in $testDirs) { + pushd $td + Write-Host "##[command]Executing go test -run "^Test" -race -v -coverprofile coverage.txt -covermode atomic $td | go-junit-report -set-exit-code > report.xml" + go test -run "^Test" -race -v -coverprofile coverage.txt -covermode atomic . | go-junit-report -set-exit-code > report.xml + # if no tests were actually run (e.g. examples) delete the coverage file so it's omitted from the coverage report + if (Select-String -path ./report.xml -pattern '' -simplematch -quiet) { + Write-Host "##[command]Deleting empty coverage file" + rm coverage.txt + } + } displayName: 'Run Tests' workingDirectory: '${{parameters.GoWorkspace}}' + env: + GO111MODULE: 'on' - pwsh: | - gocov convert ./coverage.txt > ./coverage.json - + $coverageFiles = [Collections.Generic.List[String]]@() + Get-Childitem -recurse -path $(SCOPE) -filter coverage.txt | foreach-object { + $covFile = $_.FullName + Write-Host "Adding $covFile to the list of code coverage files" + $coverageFiles.Add($covFile) + } + gocovmerge $coverageFiles > mergedCoverage.txt + gocov convert ./mergedCoverage.txt > ./coverage.json # gocov converts rely on standard input - Get-Content ./coverage.json | gocov-xml > $(Build.SourcesDirectory)/coverage.xml - Get-Content ./coverage.json | gocov-html > $(Build.SourcesDirectory)/coverage.html + Get-Content ./coverage.json | gocov-xml > ./coverage.xml + Get-Content ./coverage.json | gocov-html > ./coverage.html displayName: 'Generate Coverage XML' - workingDirectory: '${{parameters.GoWorkspace}}' + workingDirectory: '${{parameters.GoWorkspace}}sdk' - task: PublishTestResults@2 condition: succeededOrFailed() inputs: testRunner: JUnit - testResultsFiles: '$(Build.SourcesDirectory)/report.xml' + testResultsFiles: '${{parameters.GoWorkspace}}sdk/**/report.xml' testRunTitle: 'Go ${{ parameters.GoVersion }} on ${{ parameters.Image }}' + failTaskOnFailedTests: true - task: PublishCodeCoverageResults@1 condition: succeededOrFailed() inputs: codeCoverageTool: Cobertura - summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml' - additionalCodeCoverageFiles: '$(Build.SourcesDirectory)/coverage.html' + summaryFileLocation: '${{parameters.GoWorkspace}}sdk/coverage.xml' + additionalCodeCoverageFiles: '${{parameters.GoWorkspace}}sdk/coverage.html' + failIfCoverageEmpty: true diff --git a/eng/pipelines/templates/steps/set-scope.yml b/eng/pipelines/templates/steps/set-scope.yml index 09d3c71fc9e9..02d355fa69cd 100644 --- a/eng/pipelines/templates/steps/set-scope.yml +++ b/eng/pipelines/templates/steps/set-scope.yml @@ -8,7 +8,7 @@ steps: $serviceDir = "${{parameters.ServiceDirectory}}" $scope = (./eng/scripts/scoped_discover.ps1 -serviceDir "$serviceDir") - Write-Host + Write-Host $scope Write-Host "##vso[task.setvariable variable=SCOPE]$scope" displayName: 'Set Target Scope' workingDirectory: "${{ parameters.GoWorkspace }}" diff --git a/eng/scripts/get_module_dirs.ps1 b/eng/scripts/get_module_dirs.ps1 new file mode 100644 index 000000000000..5404ea45a7a7 --- /dev/null +++ b/eng/scripts/get_module_dirs.ps1 @@ -0,0 +1,15 @@ +Param( + [string] $serviceDir +) + +$modDirs = [Collections.Generic.List[String]]@() + +# find each module directory under $serviceDir +Get-Childitem -recurse -path $serviceDir -filter go.mod | foreach-object { + $cdir = $_.Directory + Write-Host "Adding $cdir to list of module paths" + $modDirs.Add($cdir) +} + +# return the list of module directories +return $modDirs diff --git a/eng/scripts/get_test_dirs.ps1 b/eng/scripts/get_test_dirs.ps1 new file mode 100644 index 000000000000..07a46f047c8b --- /dev/null +++ b/eng/scripts/get_test_dirs.ps1 @@ -0,0 +1,17 @@ +Param( + [string] $serviceDir +) + +$testDirs = [Collections.Generic.List[String]]@() + +# find each directory under $serviceDir that contains Go test files +Get-Childitem -recurse -path $serviceDir -filter *_test.go | foreach-object { + $cdir = $_.Directory + if (!$testDirs.Contains($cdir)) { + Write-Host "Adding $cdir to list of test directories" + $testDirs.Add($cdir) + } +} + +# return the list of test directories +return $testDirs diff --git a/eng/scripts/scoped_discover.ps1 b/eng/scripts/scoped_discover.ps1 index 11bccc1a031f..d7ed7e9213d5 100644 --- a/eng/scripts/scoped_discover.ps1 +++ b/eng/scripts/scoped_discover.ps1 @@ -11,4 +11,4 @@ else { $path = Resolve-Path -Path $targetDir -return "$path/..." +return "$path"