diff --git a/.github/actions/cache-docker-images/action.yaml b/.github/actions/cache-docker-images/action.yaml new file mode 100644 index 0000000000..eabb32bbfc --- /dev/null +++ b/.github/actions/cache-docker-images/action.yaml @@ -0,0 +1,51 @@ +name: Cache Docker Images +description: Cache Docker images to avoid Docker Hub rate limits + +inputs: + images: + description: "Space-separated list of Docker images to cache" + required: true + default: "docker.io/ethereum/client-go:latest docker.io/alpine:latest docker.io/library/golang:1-alpine" + cache-key-prefix: + description: "Prefix for the cache key" + required: false + default: "docker-images" + +runs: + using: "composite" + steps: + - name: Get week number + id: week + shell: bash + run: echo "num=$(date +%U)" >> $GITHUB_OUTPUT + + - name: Restore Docker image cache + id: cache-restore + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: /tmp/docker-images + key: ${{ inputs.cache-key-prefix }}-week${{ steps.week.outputs.num }}-${{ hashFiles('.github/actions/cache-docker-images/action.yaml', 'execution-specs/.github/actions/cache-docker-images/action.yaml') }} + + - name: Pull and save Docker images + shell: bash + run: | + mkdir -p /tmp/docker-images + for image in ${{ inputs.images }}; do + # Create a safe filename from image name + filename=$(echo "$image" | sed 's/[\/:]/-/g').tar.gz + if [ ! -f "/tmp/docker-images/$filename" ]; then + echo "Pulling $image..." + docker pull "$image" + echo "Saving $image to /tmp/docker-images/$filename..." + docker save "$image" | gzip > "/tmp/docker-images/$filename" + else + echo "Cache hit for $image, skipping pull" + fi + done + + - name: Save Docker image cache + if: steps.cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: /tmp/docker-images + key: ${{ inputs.cache-key-prefix }}-week${{ steps.week.outputs.num }}-${{ hashFiles('.github/actions/cache-docker-images/action.yaml', 'execution-specs/.github/actions/cache-docker-images/action.yaml') }} diff --git a/.github/actions/load-docker-images/action.yaml b/.github/actions/load-docker-images/action.yaml new file mode 100644 index 0000000000..fc375e9b16 --- /dev/null +++ b/.github/actions/load-docker-images/action.yaml @@ -0,0 +1,37 @@ +name: Load Cached Docker Images +description: Load Docker images from cache + +inputs: + cache-key-prefix: + description: "Prefix for the cache key" + required: false + default: "docker-images" + +runs: + using: "composite" + steps: + - name: Get week number + id: week + shell: bash + run: echo "num=$(date +%U)" >> $GITHUB_OUTPUT + + - name: Restore Docker image cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: /tmp/docker-images + key: ${{ inputs.cache-key-prefix }}-week${{ steps.week.outputs.num }}-${{ hashFiles('.github/actions/cache-docker-images/action.yaml', 'execution-specs/.github/actions/cache-docker-images/action.yaml') }} + + - name: Load cached Docker images + shell: bash + run: | + if [ -d /tmp/docker-images ]; then + for file in /tmp/docker-images/*.tar.gz; do + if [ -f "$file" ]; then + echo "Loading $file..." + gunzip -c "$file" | docker load + fi + done + else + echo "::error::No cached images found - cache may not exist yet. Run the cache-docker-images action first." + exit 1 + fi diff --git a/.github/workflows/hive-consume.yaml b/.github/workflows/hive-consume.yaml index cb040765f9..13b30cf16d 100644 --- a/.github/workflows/hive-consume.yaml +++ b/.github/workflows/hive-consume.yaml @@ -1,4 +1,4 @@ -name: Hive Consume Tests +name: Hive Consume on: push: @@ -7,6 +7,10 @@ on: pull_request: paths: - ".github/workflows/hive-consume.yaml" + - ".github/actions/start-hive-dev/**" + - ".github/actions/cache-docker-images/**" + - ".github/actions/load-docker-images/**" + - ".github/configs/hive/**" - "packages/testing/src/execution_testing/cli/pytest_commands/consume.py" - "packages/testing/src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-consume.ini" - "packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/**" @@ -14,6 +18,18 @@ on: - "packages/testing/src/execution_testing/fixtures/consume.py" - "packages/testing/src/execution_testing/rpc/**" workflow_dispatch: + inputs: + docker_images: + description: "Space-separated list of Docker images to cache" + required: false + default: "docker.io/ethereum/client-go:latest docker.io/alpine:latest docker.io/library/golang:1-alpine" + workflow_call: + inputs: + docker_images: + description: "Space-separated list of Docker images to cache" + required: false + type: string + default: "docker.io/ethereum/client-go:latest docker.io/alpine:latest docker.io/library/golang:1-alpine" concurrency: group: hive-consume-${{ github.workflow }}-${{ github.ref || github.run_id }} @@ -24,23 +40,36 @@ env: FIXTURES_URL: https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz jobs: + cache-docker-images: + name: Cache Docker Images + runs-on: [self-hosted-ghr, size-l-x64] + steps: + - name: Checkout execution-specs + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Cache Docker images + uses: ./.github/actions/cache-docker-images + with: + images: ${{ inputs.docker_images || 'docker.io/ethereum/client-go:latest docker.io/alpine:latest docker.io/library/golang:1-alpine' }} + test-hive: name: ${{ matrix.name }} + needs: cache-docker-images runs-on: [self-hosted-ghr, size-l-x64] strategy: fail-fast: true matrix: include: - - name: consume-engine + - name: Engine mode: simulator simulator: ethereum/eels/consume-engine - - name: consume-rlp + - name: RLP mode: simulator simulator: ethereum/eels/consume-rlp - - name: consume-sync + - name: Sync mode: simulator simulator: ethereum/eels/consume-sync - - name: dev-mode + - name: Dev Mode mode: dev consume_command: engine steps: @@ -71,6 +100,9 @@ jobs: version: ${{ vars.UV_VERSION }} python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + - name: Load cached Docker images + uses: ./execution-specs/.github/actions/load-docker-images + - name: Build hive run: | cd hive @@ -106,3 +138,4 @@ jobs: run: | uv sync --all-extras uv run consume ${{ matrix.consume_command }} --input ${{ env.FIXTURES_URL }} -k "Osaka and test_block_at_rlp_limit_with_logs" +