diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eec13a9b8cb9..de7853e2d156 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,10 @@ env: CACHE_GHA_SCOPE_IT: "integration-tests" CACHE_GHA_SCOPE_BINARIES: "binaries" CACHE_GHA_SCOPE_CROSS: "cross" + GO_VERSION: "1.18" # same as one in Dockerfile + GOTESTSUM_VERSION: "v1.8.1" # same as one in Dockerfile BUILDX_VERSION: "v0.8.2" # leave empty to use the one available on GitHub virtual environment + TESTSTAT_VERSION: "v0.1.3" jobs: base: @@ -112,23 +115,28 @@ jobs: - name: Test pkg=${{ matrix.pkg }} ; typ=${{ matrix.typ }} ; skipit=${{ matrix.skip-integration-tests }} ; worker=${{ matrix.worker }} run: | - export TESTFLAGS="-v --parallel=6 --timeout=20m" + export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.typ }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]') + export TESTFLAGS="-v --parallel=6 --timeout=30m" if [ -n "${{ matrix.worker }}" ]; then export TESTFLAGS="${TESTFLAGS} --run=//worker=${{ matrix.worker }}$" fi ./hack/test ${{ matrix.typ }} - mv ./coverage/coverage.txt ./coverage/coverage-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.typ }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]').txt env: - TEST_COVERAGE: 1 TESTPKGS: ${{ matrix.pkg }} SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }} CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} - - name: Upload coverage file - uses: actions/upload-artifact@v2 + name: Send to Codecov + uses: codecov/codecov-action@v3 + with: + directory: ./bin/testreports + - + name: Upload test reports + if: always() + uses: actions/upload-artifact@v3 with: - name: coverage - path: ./coverage + name: test-reports + path: ./bin/testreports test-s3: runs-on: ubuntu-latest @@ -170,7 +178,7 @@ jobs: name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.18 + go-version: ${{ env.GO_VERSION }} - name: Cache Go modules uses: actions/cache@v2 @@ -179,6 +187,10 @@ jobs: key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go- + - + name: Install gotestsum + run: | + go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VERSION }} - name: Go mod run: | @@ -189,45 +201,82 @@ jobs: TMPDIR: ${{ runner.temp }} SKIP_INTEGRATION_TESTS: 1 run: | - mkdir -p ./coverage - go test -coverprofile=./coverage/coverage-${{ github.job }}-${{ matrix.os }}.txt -covermode=atomic ./... + TEST_REPORT_SUFFIX=-${{ github.job }}-${{ matrix.os }} + mkdir -p ./bin/testreports + gotestsum \ + --format=standard-verbose \ + --jsonfile="./bin/testreports/go-test-report${TEST_REPORT_SUFFIX}.json" \ + --junitfile="./bin/testreports/junit-report${TEST_REPORT_SUFFIX}.xml" \ + --packages="./..." \ + -- \ + "-coverprofile" "./bin/testreports/coverage-report${TEST_REPORT_SUFFIX}.txt" \ + "-covermode" "atomic" shell: bash - - name: Upload coverage file - uses: actions/upload-artifact@v2 + name: Send to Codecov + uses: codecov/codecov-action@v3 + with: + directory: ./bin/testreports + - + name: Upload test reports + if: always() + uses: actions/upload-artifact@v3 with: - name: coverage - path: ./coverage + name: test-reports + path: ./bin/testreports - upload-coverage: + test-report: runs-on: ubuntu-latest - needs: [test, test-os] + if: always() + needs: + - test + - test-os + strategy: + fail-fast: false + matrix: + filter: + - unit + - integration-containerd + - integration-containerd-14 + - integration-containerd-15 + - integration-containerd-rootless + - integration-containerd-snapshotter-stargz + - integration-oci + - integration-oci-rootless + - integration-oci-snapshotter-stargz steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Download coverage files - uses: actions/download-artifact@v2 + name: Set up Go + uses: actions/setup-go@v3 with: - name: coverage - path: ./coverage + go-version: ${{ env.GO_VERSION }} - - name: List coverage files - uses: actions/github-script@v3 - id: files + name: Download artifacts + uses: actions/download-artifact@v3 with: - result-encoding: string - script: | - return require('fs').readdirSync('./coverage', {withFileTypes: true}) - .filter(item => !item.isDirectory()) - .map(item => `./coverage/${item.name}`) - .join(','); + name: test-reports + path: /tmp/testreports - - name: Send to Codecov - uses: codecov/codecov-action@v2 + name: Install teststat + run: | + go install github.com/vearutop/teststat@${{ env.TESTSTAT_VERSION }} + - + name: Filter test reports + run: | + filter=(! -name "*${{ matrix.filter}}.*") + if [ "${{ matrix.filter}}" = "unit" ]; then + filter=(! -name "*integrationgateway-.*" -a ! -name "*test-os-*.*") + fi + find /tmp/testreports -type f ${filter[@]} -delete + - + name: Run test-summary + uses: test-summary/action@v1 with: - files: ${{ steps.files.outputs.result }} + paths: /tmp/testreports/**/*.xml + - + name: Run teststat + run: | + teststat -markdown $(find /tmp/testreports -type f -name "*.json" -print0 | xargs -0) >> $GITHUB_STEP_SUMMARY cross: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 5b74bfefa9f7..21f152e64c57 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ bin -coverage release-out .certs .tmp diff --git a/Dockerfile b/Dockerfile index 652021a70ad7..85fb7988970e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ ARG ROOTLESSKIT_VERSION=v0.14.6 ARG CNI_VERSION=v1.1.0 ARG STARGZ_SNAPSHOTTER_VERSION=v0.11.4 ARG NERDCTL_VERSION=v0.17.1 +ARG GOTESTSUM_VERSION=v1.8.1 # ALPINE_VERSION sets version for the base layers ARG ALPINE_VERSION=3.15 @@ -184,6 +185,12 @@ RUN --mount=target=/root/.cache,type=cache \ xx-verify --static /out/containerd-stargz-grpc && \ xx-verify --static /out/ctr-remote +FROM gobuild-base AS gotestsum +ARG GOTESTSUM_VERSION +RUN --mount=target=/root/.cache,type=cache \ + GOBIN=/out/ go install "gotest.tools/gotestsum@${GOTESTSUM_VERSION}" && \ + /out/gotestsum --version + # Copy together all binaries needed for oci worker mode FROM buildkit-export AS buildkit-buildkitd.oci_only COPY --link --from=buildkitd.oci_only /usr/bin/buildkitd.oci_only /usr/bin/ @@ -241,6 +248,7 @@ RUN apk add --no-cache shadow shadow-uidmap sudo vim iptables fuse curl \ ENV BUILDKIT_INTEGRATION_CONTAINERD_EXTRA="containerd-1.4=/opt/containerd-alt-14/bin,containerd-1.5=/opt/containerd-alt-15/bin" ENV BUILDKIT_INTEGRATION_SNAPSHOTTER=stargz ENV CGO_ENABLED=0 +COPY --link --from=gotestsum /out/gotestsum /usr/bin/ COPY --link --from=stargz-snapshotter /out/* /usr/bin/ COPY --link --from=rootlesskit /rootlesskit /usr/bin/ COPY --link --from=containerd-alt-14 /out/containerd* /opt/containerd-alt-14/bin/ diff --git a/hack/test b/hack/test index 7b6ea11a8d1f..b378091b235e 100755 --- a/hack/test +++ b/hack/test @@ -8,7 +8,7 @@ set -eu -o pipefail : ${TEST_DOCKERFILE=} : ${TEST_DOCKERD=} : ${TEST_DOCKERD_BINARY=$(which dockerd)} -: ${TEST_COVERAGE=} +: ${TEST_REPORT_SUFFIX=} : ${TEST_KEEP_CACHE=} : ${DOCKERFILE_RELEASES=} : ${BUILDKIT_WORKER_RANDOM=} @@ -50,14 +50,11 @@ done iid="buildkit-tests" iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX) -coverageVol="" -coverageFlags="" -if [ "$TEST_COVERAGE" = "1" ]; then - covdir="$(pwd)/coverage" - mkdir -p "$covdir" - coverageVol="-v $covdir:/coverage" - coverageFlags="-coverprofile=/coverage/coverage.txt -covermode=atomic" -fi +testReportsDir="$(pwd)/bin/testreports" +mkdir -p "$testReportsDir" +testReportsVol="-v $testReportsDir:/testreports" +gotestsumArgs="--format=standard-verbose --jsonfile=/testreports/go-test-report$TEST_REPORT_SUFFIX.json --junitfile=/testreports/junit-report$TEST_REPORT_SUFFIX.xml" +gotestArgs="-coverprofile=/testreports/coverage-report$TEST_REPORT_SUFFIX.txt -covermode=atomic" buildxCmd build $cacheFromFlags \ --target "integration-tests" \ @@ -70,7 +67,7 @@ if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then fi if [ "$TEST_INTEGRATION" == 1 ]; then - cid=$(docker create --rm -v /tmp $coverageVol --volumes-from=$cacheVolume -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...}) + cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v}) if [ "$TEST_DOCKERD" = "1" ]; then docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd fi @@ -110,7 +107,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then if [ -s $tarout ]; then if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$GITHUB_ACTIONS" = "true" ]; then - cid=$(docker create -v /tmp $coverageVol --rm --privileged --volumes-from=$cacheVolume -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test $coverageFlags --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile) + cid=$(docker create -v /tmp $testReportsVol --rm --privileged --volumes-from=$cacheVolume -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid gotestsum $gotestsumArgs --packages=./frontend/dockerfile -- $gotestArgs --count=1 -tags "$buildtags" ${TESTFLAGS:--v}) docker cp $tarout $cid:/$release.tar if [ "$TEST_DOCKERD" = "1" ]; then docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd