ci: improve caching

.github/actions/docker/action.yaml

@@ -22,12 +22,6 @@ inputs:
     description: User name to use when pushing images to Docker Hub
   dockerhub_token:
     description: Docker Hub token to use
-  cache_mounts:
-    description: Load cache mounts cache
-    default: |
-      cargo_registry_index
-      cargo_registry_cache
-      cargo_git
   cargo_profile:
     description: Cargo build profile, i.e release or dev
     default: dev
@@ -69,12 +63,12 @@ runs:
           [worker.oci]
             gc = false
 
-    - name: Load Docker mount cache
-      uses: dcginfra/buildkit-cache-dance/inject@s5cmd
-      if: ${{ inputs.cache_mounts != '' }}
-      with:
-        bucket: ${{ inputs.bucket }}
-        mounts: ${{ inputs.cache_mounts }}
+    #    - name: Load Docker mount cache
+    #      uses: dcginfra/buildkit-cache-dance/inject@s5cmd
+    #      if: ${{ inputs.cache_mounts != '' }}
+    #      with:
+    #        bucket: ${{ inputs.bucket }}
+    #        mounts: ${{ inputs.cache_mounts }}
 
     - name: Set Docker tags and labels from image tag
       id: docker_meta
@@ -85,37 +79,109 @@ runs:
           type=semver,pattern={{version}},value=${{inputs.image_version}},enable=${{ inputs.image_version != '' }}
           type=sha,format=long,enable=${{ inputs.image_version == '' }}
 
-    - name: Configure docker layer cache
-      uses: ./.github/actions/s3-layer-cache-settings
-      id: layer_cache_settings
+#    - name: Configure docker layer cache
+#      uses: ./.github/actions/s3-layer-cache-settings
+#      id: layer_cache_settings
+#      with:
+#        name: ${{ inputs.image_name }}
+#        bucket: ${{ inputs.bucket }}
+
+    - name: Cargo cache for Docker
+      uses: actions/cache@v4
+      id: cargo-cache
+      with:
+        path: |
+          ../cargo-cache-registry-index
+          ../cargo-cache-registry-cache
+          ../cargo-cache-git-db
+        key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
+        restore-keys: |
+          - ${{ runner.os }}-cargo-
+
+    - name: Inject cargo cache into docker
+      uses: reproducible-containers/[email protected]
+      with:
+        cache-map: |
+          {
+            "../cargo-cache-registry-index": {
+              "target": "/root/.cargo/registry/index",
+              "id": "cargo_registry_index"
+            },
+            "../cargo-cache-registry-cache": {
+              "target": "/root/.cargo/registry/cache",
+              "id": "cargo_registry_cache"
+            },
+            "../cargo-cache-git-db": {
+              "target": "/root/.cargo/git/db",
+              "id": "cargo_git"
+            }
+          }
+        skip-extraction: ${{ steps.cargo-cache.outputs.cache-hit }}
+
+    - name: Yarn unplugged cache for Docker
+      uses: actions/cache@v4
+      id: yarn-cache
+      with:
+        path: ../yarn-unplugged-cache
+        key: ${{ inputs.platform }}-yarn-unplugged-${{ hashFiles('yarn.lock') }}
+        restore-keys: |
+          - ${{ inputs.platform }}-yarn-unplugged-
+
+    - name: Set arch
+      id: arch
+      uses: actions/github-script@v6
+      with:
+        result-encoding: 'string'
+        script: return '${{ inputs.platform }}'.replace('linux/', '');
+
+    - name: Inject cargo cache into docker
+      uses: reproducible-containers/[email protected]
+      with:
+        cache-map: |
+          {
+            "../yarn-unplugged-cache": {
+              "target": "/tmp/unplugged",
+              "id": "unplugged_${{ steps.arch.outputs.result }}"
+            }
+          }
+        skip-extraction: ${{ steps.yarn-cache.outputs.cache-hit }}
+
+    - name: Export actions cache vars
+      uses: actions/github-script@v7
       with:
-        name: ${{ inputs.image_name }}
-        bucket: ${{ inputs.bucket }}
+        script: |
+          core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+          core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
 
     - name: Build and push Docker image ${{ inputs.image }}
       id: docker_build
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       with:
         context: .
         builder: ${{ steps.buildx.outputs.name }}
         target: ${{ inputs.target }}
         labels: ${{ steps.docker_meta.outputs.labels }}
+        push: ${{ inputs.push_tags }}
         tags: ${{ inputs.push_tags == 'true' && steps.docker_meta.outputs.tags || '' }}
         platforms: ${{ inputs.platform }}
         build-args: |
           CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }}
           RUSTC_WRAPPER=sccache
-          SCCACHE_BUCKET=${{ inputs.bucket }}
-          SCCACHE_REGION=${{ inputs.region }}
-          SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache
-          AWS_ACCESS_KEY_ID=${{ inputs.aws_access_key_id }}
-          AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }}
-          AWS_REGION=${{ inputs.region }}
-        cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }}
-        cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }}
+          SCCACHE_GHA_ENABLED=true
+          ACTIONS_CACHE_URL=${{ env.ACTIONS_CACHE_URL }}
+          ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
+#          SCCACHE_BUCKET=${{ inputs.bucket }}
+#          SCCACHE_REGION=${{ inputs.region }}
+#          SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache
+#          AWS_ACCESS_KEY_ID=${{ inputs.aws_access_key_id }}
+#          AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }}
+#          AWS_REGION=${{ inputs.region }}
+#        cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }}
+#        cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
         outputs: type=image,name=${{ inputs.image_org }}/${{ inputs.image_name }},push-by-digest=${{ inputs.push_tags != 'true' }},name-canonical=true,push=true
 
-# TODO: This is doesn't work
 #    - name: Save Docker mount cache
 #      uses: dcginfra/buildkit-cache-dance/extract@s5cmd
 #      if: ${{ inputs.cache_mounts != '' }}

.github/actions/librocksdb/action.yaml

@@ -22,13 +22,19 @@ runs:
   using: composite
   steps:
     # Cache librocksdb using s3 bucket
-    - name: Restore cached librocksdb from S3
-      id: librocksdb-cache
-      uses: strophy/actions-cache@opendal-update
+#    - name: Restore cached librocksdb from S3
+#      id: librocksdb-cache
+#      uses: strophy/actions-cache@opendal-update
+#      with:
+#        bucket: ${{ inputs.bucket }}
+#        path: /opt/rocksdb
+#        key: librocksdb/${{ inputs.version }}/${{ runner.os }}/${{ runner.arch }}
+
+    - name: Restore cached librocksdb
+      uses: actions/cache@v4
       with:
-        bucket: ${{ inputs.bucket }}
-        path: /opt/rocksdb
         key: librocksdb/${{ inputs.version }}/${{ runner.os }}/${{ runner.arch }}
+        path: /opt/rocksdb
 
     - if: ${{ steps.librocksdb-cache.outputs.cache-hit != 'true' || inputs.force == 'true' }}
       shell: bash

.github/actions/local-network/action.yaml

@@ -11,13 +11,20 @@ runs:
     - name: Setup Node.JS
       uses: ./.github/actions/nodejs
 
+#    - name: Restore JS build artifacts
+#      uses: strophy/actions-cache@opendal-update
+#      with:
+#        bucket: multi-runner-cache-x1xibo9c
+#        root: actions-cache
+#        path: build-js-artifacts-${{ github.sha }}.tar
+#        key: build-js-artifacts/${{ github.sha }}
+
     - name: Restore JS build artifacts
-      uses: strophy/actions-cache@opendal-update
+      uses: actions/cache/restore@v4
       with:
-        bucket: multi-runner-cache-x1xibo9c
-        root: actions-cache
+        key: build-js-artifacts-${{ github.sha }}
         path: build-js-artifacts-${{ github.sha }}.tar
-        key: build-js-artifacts/${{ github.sha }}
+        fail-on-cache-miss: 'true'
 
     - name: Unpack JS build artifacts archive
       shell: bash
@@ -33,12 +40,22 @@ runs:
       run: echo "HOME=$HOME" >> $GITHUB_ENV
 
     # TODO: Use upload artifacts action instead
+#    - name: Restore local network data
+#      id: local-network-data
+#      uses: strophy/actions-cache/restore@opendal-update
+#      with:
+#        bucket: multi-runner-cache-x1xibo9c
+#        root: local-network-data
+#        path: |
+#          ${{ env.HOME }}/.dashmate
+#          **/.env
+#          dashmate_volumes_dump
+#        key: local-network-volumes/${{ steps.dashmate-fingerprint.outputs.sha }}
+
     - name: Restore local network data
       id: local-network-data
-      uses: strophy/actions-cache/restore@opendal-update
+      uses: actions/cache/restore@v4
       with:
-        bucket: multi-runner-cache-x1xibo9c
-        root: local-network-data
         path: |
           ${{ env.HOME }}/.dashmate
           **/.env
@@ -71,11 +88,21 @@ runs:
       shell: bash
       if: steps.local-network-data.outputs.cache-hit != 'true'
 
+#    - name: Save local network data
+#      uses: strophy/actions-cache/save@opendal-update
+#      with:
+#        bucket: multi-runner-cache-x1xibo9c
+#        root: local-network-data
+#        path: |
+#          ${{ env.HOME }}/.dashmate
+#          **/.env
+#          dashmate_volumes_dump
+#        key: local-network-volumes/${{ steps.dashmate-fingerprint.outputs.sha }}
+#      if: steps.local-network-data.outputs.cache-hit != 'true'
+
     - name: Save local network data
-      uses: strophy/actions-cache/save@opendal-update
+      uses: actions/cache/save@v4
       with:
-        bucket: multi-runner-cache-x1xibo9c
-        root: local-network-data
         path: |
           ${{ env.HOME }}/.dashmate
           **/.env

.github/actions/nodejs/action.yaml

@@ -17,11 +17,20 @@ runs:
       shell: bash
       run: npm config set audit false
 
-    - name: Cache NPM build artifacts (S3 bucket cache)
-      uses: strophy/actions-cache@opendal-update
+#    - name: Cache NPM build artifacts (S3 bucket cache)
+#      uses: strophy/actions-cache@opendal-update
+#      with:
+#        bucket: multi-runner-cache-x1xibo9c
+#        root: actions-cache
+#        path: |
+#          .yarn/unplugged
+#        key: ${{ runner.os }}/yarn/unplugged/${{ runner.arch }}/${{ hashFiles('yarn.lock') }}
+#        restore-keys: |
+#          ${{ runner.os }}/yarn/unplugged/${{ runner.arch }}/
+
+    - name: Cache NPM build artifacts
+      uses: actions/cache@v4
       with:
-        bucket: multi-runner-cache-x1xibo9c
-        root: actions-cache
         path: |
           .yarn/unplugged
         key: ${{ runner.os }}/yarn/unplugged/${{ runner.arch }}/${{ hashFiles('yarn.lock') }}

.github/actions/rust/action.yaml

@@ -99,12 +99,25 @@ runs:
       shell: bash
       run: echo "HOME=$HOME" >> $GITHUB_ENV
 
-    - name: Cache cargo registry (S3 bucket cache)
-      uses: strophy/actions-cache@opendal-update
+#    - name: Cache cargo registry (S3 bucket cache)
+#      uses: strophy/actions-cache@opendal-update
+#      if: inputs.cache == 'true'
+#      with:
+#        bucket: multi-runner-cache-x1xibo9c
+#        root: actions-cache
+#        path: |
+#          ${{ env.HOME }}/.cargo/registry/index
+#          ${{ env.HOME }}/.cargo/registry/cache
+#          ${{ env.HOME }}/.cargo/git
+#        key: ${{ runner.os }}/cargo/registry/${{ hashFiles('**/Cargo.lock') }}
+#        restore-keys: |
+#          ${{ runner.os }}/cargo/registry/${{ hashFiles('**/Cargo.lock') }}
+#          ${{ runner.os }}/cargo/registry/
+
+    - name: Cache cargo registry
+      uses: actions/cache@v4
       if: inputs.cache == 'true'
       with:
-        bucket: multi-runner-cache-x1xibo9c
-        root: actions-cache
         path: |
           ${{ env.HOME }}/.cargo/registry/index
           ${{ env.HOME }}/.cargo/registry/cache
@@ -114,6 +127,7 @@ runs:
           ${{ runner.os }}/cargo/registry/${{ hashFiles('**/Cargo.lock') }}
           ${{ runner.os }}/cargo/registry/
 
+
     # TODO: Move to AMI and build every day
     - name: Install clang
       id: deps-clang