ci: change s3 cache provider to optimize costs (#2344)

Author: lklimek

.github/actions/aws_credentials/action.yaml

@@ -0,0 +1,49 @@
+---
+name: "aws_credentials"
+description: |
+  Configure .aws/credentials file with provided access key ID and secret access key.
+
+  This action creates a credentials file in ${HOME}/.aws/credentials with the provided access key ID and secret access key.
+  It also sets AWS_PROFILE and AWS_SHARED_CREDENTIALS_FILE environment variables to use this profile.
+
+  It can conflict with other actions that define AWS credentials or set AWS_PROFILE env variable.
+  Explicitly set AWS_PROFILE=sccache and unset AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in case
+  of conflicting settings.
+inputs:
+  access_key_id:
+    description: Access key ID
+    required: true
+  secret_access_key:
+    description: Secret access key
+    required: true
+  profile:
+    description: AWS profile to use; set AWS_PROFILE env variable to use this profile
+    default: "default"
+
+runs:
+  using: composite
+  steps:
+    - name: Configure AWS credentials
+      shell: bash
+      run: |
+        mkdir -p "${HOME}/.aws"
+        cat >> ${HOME}/.aws/credentials << EOF
+        [${{ inputs.profile }}]
+        aws_access_key_id=${{ inputs.access_key_id }}
+        aws_secret_access_key=${{ inputs.secret_access_key }}
+        EOF
+        chmod -R go-rwx ${HOME}/.aws
+
+    - name: Set env variables
+      shell: bash
+      run: |
+        # Exit on any error  
+        set -euo pipefail  
+        # Validate AWS_PROFILE is not empty  
+        if [ -z "${{ inputs.profile }}" ]; then  
+          echo "Error: AWS_PROFILE cannot be empty"
+          exit 1
+        fi
+        # Export variables        
+        echo "AWS_PROFILE=${{ inputs.profile }}" >> $GITHUB_ENV
+        echo "AWS_SHARED_CREDENTIALS_FILE=${HOME}/.aws/credentials" >> $GITHUB_ENV

.github/actions/aws_ecr_login/action.yaml

@@ -0,0 +1,42 @@
+---
+# Login to AWS ECR
+name: "aws_ecr_login"
+description: "Login to AWS ECR to store Docker containers"
+inputs:
+  aws_account_id:
+    description: AWS account ID (AWS_ACCOUNT_ID)
+    required: true
+  aws_access_key_id:
+    description: Access key ID (AWS_ACCESS_KEY_ID)
+    required: true
+  aws_secret_access_key:
+    description: Secret access key (AWS_SECRET_ACCESS_KEY)
+    required: true
+  aws_region:
+    description: AWS region to use (AWS_REGION)
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Configure AWS credentials and bucket region
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ inputs.aws_access_key_id }}
+        aws-secret-access-key: ${{ inputs.aws_secret_access_key }}
+        aws-region: ${{ inputs.aws_region }}
+
+    - name: Login to ECR
+      run: |
+        aws ecr get-login-password \
+          --region ${{ inputs.aws_region }} | docker login --username AWS --password-stdin ${{ inputs.aws_account_id }}.dkr.ecr.${{ inputs.aws_region  }}.amazonaws.com
+      shell: bash
+
+    # Unset AWS credentials to avoid conflicts, as we prefer credentials from ~/.aws/credentials to authenticate
+    - name: Unset AWS credentials to avoid conflicts
+      shell: bash
+      run: |
+        echo AWS_DEFAULT_REGION='' >> $GITHUB_ENV
+        echo AWS_REGION='' >> $GITHUB_ENV
+        echo AWS_ACCESS_KEY_ID='' >> $GITHUB_ENV
+        echo AWS_SECRET_ACCESS_KEY='' >> $GITHUB_ENV

.github/actions/docker/action.yaml

@@ -1,6 +1,13 @@
 ---
 name: "Build and push docker image"
-description: "Build and push docker image by digest with Rust caching"
+description: |
+  Build and push docker image by digest with extensive caching.
+
+  This action builds and pushes a Docker image to Docker Hub.
+  It uses caching for Rust dependencies and Docker layers.
+  It also provides sccache settings to the docker builder for caching Rust compilation.
+
+  Layers cache and sccache will use the same credentials and S3 bucket, but different prefixes.
 inputs:
   image_name:
     description: Name of image in Docker Hub, like `drive`
@@ -25,21 +32,24 @@ inputs:
   cargo_profile:
     description: Cargo build profile, i.e release or dev
     default: dev
-  bucket:
-    description: S3 bucket to use for caching, must match runner define in `runs-on`
-    default: multi-runner-cache-x1xibo9c
-  region:
+  cache_bucket:
+    description: S3 bucket to use for caching (both sccache and layer cache)
+    required: true
+  cache_region:
     description: S3 bucket region
     required: true
-  aws_access_key_id:
-    description: AWS access key ID
+  cache_endpoint:
+    description: S3 endpoint to use for caching
+    required: true
+  cache_access_key_id:
+    description: Access key ID for s3 cache
     required: true
-  aws_secret_access_key:
-    description: AWS secret access key
+  cache_secret_access_key:
+    description: Secret access key for s3 cache
     required: true
   cache_to_name:
-    description: 'Save cache to name manifest (should be used only on default branch)'
-    default: 'false'
+    description: "Save cache to name manifest (should be used only on default branch)"
+    default: "false"
 outputs:
   digest:
     value: ${{ steps.docker_build.outputs.digest }}
@@ -80,9 +90,13 @@ runs:
       id: layer_cache_settings
       with:
         name: ${{ inputs.image_name }}
-        region: ${{ inputs.region }}
-        bucket: ${{ inputs.bucket }}
+        region: ${{ inputs.cache_region }}
+        bucket: ${{ inputs.cache_bucket }}
+        endpoint: ${{ inputs.cache_endpoint }}
+        prefix: "cache-layers/${{ inputs.platform }}/"
         cache_to_name: ${{ inputs.cache_to_name }}
+        s3_access_key_id: ${{ inputs.cache_access_key_id }}
+        s3_secret_access_key: ${{ inputs.cache_secret_access_key }}
 
     - name: Set HOME variable to github context
       shell: bash
@@ -133,7 +147,7 @@ runs:
       id: arch
       uses: actions/github-script@v6
       with:
-        result-encoding: 'string'
+        result-encoding: "string"
         script: return '${{ inputs.platform }}'.replace('linux/', '');
 
     - name: Inject cargo cache into docker
@@ -148,9 +162,24 @@ runs:
           }
         skip-extraction: ${{ steps.yarn-cache.outputs.cache-hit }}
 
+    - name: Configure sccache settings
+      uses: ./.github/actions/sccache
+      id: sccache
+      with:
+        bucket: ${{ inputs.cache_bucket }}
+        region: ${{ inputs.cache_region }}
+        endpoint: ${{ inputs.cache_endpoint }}
+        access_key_id: ${{ inputs.cache_access_key_id }}
+        secret_access_key: ${{ inputs.cache_secret_access_key }}
+        platform: ${{ inputs.platform }}
+        install: false
+
     - name: Build and push Docker image ${{ inputs.image }}
       id: docker_build
       uses: docker/build-push-action@v6
+      env:
+        # AWS profile to be used by layer cache; sccache settings are passed via build-args
+        AWS_PROFILE: ${{ steps.layer_cache_settings.outputs.aws_profile }}
       with:
         context: .
         builder: ${{ steps.buildx.outputs.name }}
@@ -159,14 +188,9 @@ runs:
         push: ${{ inputs.push_tags }}
         tags: ${{ inputs.push_tags == 'true' && steps.docker_meta.outputs.tags || '' }}
         platforms: ${{ inputs.platform }}
-        build-args: |
-          CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }}
-          RUSTC_WRAPPER=sccache
-          SCCACHE_BUCKET=${{ inputs.bucket }}
-          SCCACHE_REGION=${{ inputs.region }}
-          SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache
-          AWS_ACCESS_KEY_ID=${{ inputs.aws_access_key_id }}
-          AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }}
+        secret-files: |
+          AWS=${{ env.HOME }}/.aws/credentials
+        build-args: ${{ steps.sccache.outputs.env_vars }}
         cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }}
         cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }}
         outputs: type=image,name=${{ inputs.image_org }}/${{ inputs.image_name }},push-by-digest=${{ inputs.push_tags != 'true' }},name-canonical=true,push=true

.github/actions/librocksdb/action.yaml

@@ -9,10 +9,6 @@ inputs:
     description: RocksDB version, eg. "8.10.2"
     required: false
     default: "8.10.2"
-  bucket:
-    description: S3 bucket to use for caching
-    required: false
-    default: multi-runner-cache-x1xibo9c
   force:
     description: Force rebuild
     required: false

.github/actions/rust/action.yaml

@@ -31,7 +31,7 @@ runs:
         fi
 
         echo "TOOLCHAIN_VERSION=$TOOLCHAIN_VERSION" >> $GITHUB_ENV
-        echo "::set-output name=version::$TOOLCHAIN_VERSION"
+        echo "version=$TOOLCHAIN_VERSION"  >> $GITHUB_OUTPUT
 
     - uses: dtolnay/rust-toolchain@master
       name: Install Rust toolchain
@@ -82,12 +82,6 @@ runs:
         echo "PROTOC=${HOME}/.local/bin/protoc" >> $GITHUB_ENV
         export PATH="${PATH}:${HOME}/.local/bin"
 
-    - name: Install sccache-cache
-      uses: mozilla-actions/[email protected]
-      with:
-        version: "v0.8.2" # Must be the same as in Dockerfile
-      if: inputs.cache == 'true'
-
     - name: Set HOME variable to github context
       shell: bash
       run: echo "HOME=$HOME" >> $GITHUB_ENV

.github/actions/s3-layer-cache-settings/action.yaml

@@ -1,45 +1,66 @@
-name: 'Get S3 Docker Layer Cache settings'
+name: "Get S3 Docker Layer Cache settings"
 description: |
   This action generates string with s3-based cache configuration for docker buildx.
   It defines three manifests:
     - name and current commit to hit all builds for this commit (restart) with this name
     - name and head ref to hit all builds for this branch with this name
     - just name to hit all builds for this name
 
+  To correcly use caching, ensure buildx has AWS_PROFILE environment set to value of `aws_profile` output.
+
 inputs:
   name:
-    description: 'Cache key name will be used as a prefix for all docker image manifests'
+    description: "Cache key name will be used as a prefix for all docker image manifests"
     required: true
   head_ref:
-    description: 'Head ref for an additional manifest to hit all builds for this head'
+    description: "Head ref for an additional manifest to hit all builds for this head"
     default: ${{ github.ref }}
   region:
     description: S3 region
     required: true
   bucket:
     description: S3 bucket name
     required: true
+  endpoint:
+    description: S3 endpoint to use for caching
+    required: false
   prefix:
     description: S3 key prefix
-    default: 'cache-layers/'
+    default: "cache-layers/"
+  s3_access_key_id:
+    description: Access key ID for S3 cache
+    required: true
+  s3_secret_access_key:
+    description: Secret access key for S3 cache
+    required: true
   mode:
     description: Cache mode
     default: max
   cache_to_name:
-    description: 'Save cache to name manifest (should be used only on default branch)'
-    default: 'false'
+    description: "Save cache to name manifest (should be used only on default branch)"
+    default: "false"
 
 outputs:
   cache_to:
-    description: 'String with s3-based cache configuration for docker buildx cache-to option'
+    description: "String with s3-based cache configuration for docker buildx cache-to option"
     value: ${{ steps.script.outputs.cache_to }}
   cache_from:
-    description: 'String with s3-based cache configuration for docker buildx cache-from option'
+    description: "String with s3-based cache configuration for docker buildx cache-from option"
     value: ${{ steps.script.outputs.cache_from }}
+  aws_profile:
+    description: "AWS profile to use for s3 cache, to set inside AWS_PROFILE env var"
+    value: layers
 
 runs:
   using: composite
   steps:
+    - name: Configure AWS credentials for s3 layers
+      uses: ./.github/actions/aws_credentials
+      with:
+        access_key_id: ${{ inputs.s3_access_key_id }}
+        secret_access_key: ${{ inputs.s3_secret_access_key }}
+        profile: "layers"
+
     - uses: actions/github-script@v6
       id: script
       with:
@@ -49,6 +70,7 @@ runs:
             region: '${{ inputs.region }}',
             bucket: '${{ inputs.bucket }}',
             prefix: '${{ inputs.prefix }}',
+            endpoint_url: '${{ inputs.endpoint }}',
           };
 
           const settingsString = Object.entries(settings)
@@ -59,7 +81,7 @@ runs:
           const sanitizedHeadRef = '${{ inputs.head_ref }}'.replace(/[^a-zA-Z0-9]/g, '-');
 
           const shaManifestName = '${{ inputs.name }}_sha_${{ github.sha }}';
-          const headRefManifestName = '${{ inputs.name }}_tag_${ sanitizedHeadRef }';
+          const headRefManifestName = '${{ inputs.name }}_tag_' + sanitizedHeadRef;
 
           const cacheFromManifestNames = [
             shaManifestName,