From 28876687706b2d8d5d55fc53fb47e6fd7b1cb8cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=C5=9Alizak?= Date: Thu, 27 Oct 2022 15:49:30 +0200 Subject: [PATCH 1/2] Stop docs job from caching incomplete Maven repositories `setup-java` action caches the local Maven repo once a job succeeds. The cache key for Maven depends only on the hash of all pom.xml files. By uploading caches from a job which only builds one module we are forcing Maven to download missing dependencies during each build. --- .github/workflows/docs.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7c6903103e8c..de7fdb10f30d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -61,6 +61,10 @@ jobs: run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" $RETRY $MAVEN install -B --strict-checksums -V -T C1 -DskipTests -P ci -am -pl ':trino-docs' + - name: Clean local Maven repo + # Avoid creating a cache entry because this job doesn't download all dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: rm -rf ~/.m2/repository test-docs: needs: path-filters @@ -121,3 +125,7 @@ jobs: path: | **/surefire-reports/TEST-*.xml retention-days: ${{ env.TEST_REPORT_RETENTION_DAYS }} + - name: Clean local Maven repo + # Avoid creating a cache entry because this job doesn't download all dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: rm -rf ~/.m2/repository From 54b8b8b43645def9dd4a0699507e1eac1148e3bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=C5=9Alizak?= Date: Fri, 28 Oct 2022 11:22:47 +0200 Subject: [PATCH 2/2] Update Maven dependency cache in `maven-checks` job When uploading a new cache entry it's important that the entry be complete, to avoid the need of downloading more dependencies during build. This change explicitly downloads all dependencies in `maven-checks` job which already builds all modules, to minimize additional work. --- .github/workflows/ci.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a7e78df556c..50d23e475e60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,14 +59,17 @@ jobs: cache: 'maven' - name: Configure Problem Matchers run: echo "::add-matcher::.github/problem-matcher.json" + - name: Download all Trino dependencies to make sure cache is complete + run: | + $RETRY $MAVEN dependency:go-offline - name: Maven Checks run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $RETRY $MAVEN clean install -B --strict-checksums -V -T C1 -DskipTests -P ci -pl '!:trino-server-rpm' + $MAVEN clean install -B --strict-checksums -V -T C1 -DskipTests -P ci -pl '!:trino-server-rpm' - name: Test Server RPM run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $RETRY $MAVEN verify -B --strict-checksums -P ci -pl :trino-server-rpm + $MAVEN verify -B --strict-checksums -P ci -pl :trino-server-rpm - name: Clean Maven Output run: $MAVEN clean -pl '!:trino-server,!:trino-cli' - uses: docker/setup-qemu-action@v2 @@ -74,7 +77,7 @@ jobs: platforms: arm64,ppc64le - name: Test Docker Image run: core/docker/build.sh - - name: Clean local Maven repo + - name: Remove Trino from local Maven repo to avoid caching it # Avoid caching artifacts built in this job, cache should only include dependencies if: steps.cache.outputs.cache-hit != 'true' run: rm -rf ~/.m2/repository/io/trino/trino-* @@ -116,6 +119,8 @@ jobs: git rebase --verbose --exec "$MAVEN package ${MAVEN_COMPILE_COMMITS} ${MAVEN_GIB} || (git status && false)" $( git merge-base HEAD refs/remotes/origin/${{ github.event.pull_request.base.ref }} ) # The local repo is broken at this point, and should be abandoned - name: Clean local Maven repo + # Avoid creating a cache entry because this job doesn't download all dependencies + if: steps.cache.outputs.cache-hit != 'true' run: rm -rf ~/.m2/repository error-prone-checks: @@ -146,6 +151,10 @@ jobs: # The main Error Prone run $MAVEN ${MAVEN_TEST} -T C1 clean verify -DskipTests -P gib,errorprone-compiler \ -pl '!:trino-docs,!:trino-server,!:trino-server-rpm' + - name: Clean local Maven repo + # Avoid creating a cache entry because this job doesn't download all dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: rm -rf ~/.m2/repository web-ui-checks: runs-on: ubuntu-latest