diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 734dc51384f2..88466267e335 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -31,6 +31,8 @@ env:
   # This value should be greater than the time taken for the longest image pull.
   TESTCONTAINERS_PULL_PAUSE_TIMEOUT: 600
   TEST_REPORT_RETENTION_DAYS: 5
+  # used by actions/cache to retry the download after this time: https://github.com/actions/cache/blob/main/workarounds.md#cache-segment-restore-timeout
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
 
 # Cancel previous PR builds.
 concurrency:
@@ -77,9 +79,13 @@ jobs:
           platforms: arm64,ppc64le
       - name: Test Docker Image
         run: core/docker/build.sh
+      - name: Clean local Maven repo
+        # Avoid creating a cache entry except when pushing to the default branch
+        if: steps.cache.outputs.cache-hit != 'true' || !(github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == github.event.repository.default_branch)
+        run: rm -rf ~/.m2/repository
       - name: Remove Trino from local Maven repo to avoid caching it
         # Avoid caching artifacts built in this job, cache should only include dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
+        if: steps.cache.outputs.cache-hit != 'true' && github.event_name == 'push' && github.ref_type == 'branch' && github.ref_name == github.event.repository.default_branch
         run: rm -rf ~/.m2/repository/io/trino/trino-*
 
   check-commits: