From 3af61bcc3e9537511313de2b4cfdeed8b27565e3 Mon Sep 17 00:00:00 2001 From: Guruprasad Kamath Date: Fri, 18 Jul 2025 13:28:58 +0200 Subject: [PATCH 1/6] re-structure ci --- .github/workflows/test.yaml | 132 +++++++++++++++++++++++++++++++----- 1 file changed, 114 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8274d6369e..9e3c581b7f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,53 +10,149 @@ on: pull_request: jobs: - build: + static: runs-on: [self-hosted-ghr, size-xl-x64] - strategy: - matrix: - py: [ "3.11", "pypy3.11" ] - steps: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install Rust uses: actions-rs/toolchain@v1 with: - toolchain: "1.69.0" - override: true - + toolchain: "1.69.0" + override: true - name: Setup Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.py }} - + python-version: "3.11" - name: Install Tox and any other packages run: | DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config pip install 'tox>=4.11,<5' requests - - name: Download Geth and add to $PATH run: | mkdir -p $GITHUB_WORKSPACE/bin $GITHUB_WORKSPACE/scripts/download_geth_linux.py --dir $GITHUB_WORKSPACE/bin echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - name: Run static checks + run: tox -e static - - name: Run Tox (CPython) - if: "${{ !startsWith(matrix.py, 'pypy') }}" - run: tox -e static,py3_eest,optimized,py3 + py3_eest: + runs-on: [self-hosted-ghr, size-xl-x64] + needs: static + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: "1.69.0" + override: true + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install Tox and any other packages + run: | + DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config + pip install 'tox>=4.11,<5' requests + - name: Download Geth and add to $PATH + run: | + mkdir -p $GITHUB_WORKSPACE/bin + $GITHUB_WORKSPACE/scripts/download_geth_linux.py --dir $GITHUB_WORKSPACE/bin + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - name: Run py3_eest tests + run: tox -e py3_eest - - name: Run Tox (PyPy) - if: "${{ startsWith(matrix.py, 'pypy') }}" + pypy3: + runs-on: [self-hosted-ghr, size-xl-x64] + needs: static + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: "1.69.0" + override: true + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "pypy3.11" + - name: Install Tox and any other packages + run: | + DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config + pip install 'tox>=4.11,<5' requests + - name: Download Geth and add to $PATH + run: | + mkdir -p $GITHUB_WORKSPACE/bin + $GITHUB_WORKSPACE/scripts/download_geth_linux.py --dir $GITHUB_WORKSPACE/bin + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - name: Run pypy3 tests run: tox -e pypy3 env: PYPY_GC_MAX: "10G" + py3: + runs-on: [self-hosted-ghr, size-xl-x64] + needs: [py3_eest, pypy3] + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: "1.69.0" + override: true + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install Tox and any other packages + run: | + DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config + pip install 'tox>=4.11,<5' requests + - name: Download Geth and add to $PATH + run: | + mkdir -p $GITHUB_WORKSPACE/bin + $GITHUB_WORKSPACE/scripts/download_geth_linux.py --dir $GITHUB_WORKSPACE/bin + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - name: Run py3 tests + run: tox -e py3 - name: Upload coverage reports to Codecov - if: "${{ !startsWith(matrix.py, 'pypy') }}" uses: codecov/codecov-action@v5 with: files: .tox/coverage.xml flags: unittests token: ${{ secrets.CODECOV_TOKEN }} + + optimized: + runs-on: [self-hosted-ghr, size-xl-x64] + needs: [py3_eest, pypy3] + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: "1.69.0" + override: true + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install Tox and any other packages + run: | + DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config + pip install 'tox>=4.11,<5' requests + - name: Download Geth and add to $PATH + run: | + mkdir -p $GITHUB_WORKSPACE/bin + $GITHUB_WORKSPACE/scripts/download_geth_linux.py --dir $GITHUB_WORKSPACE/bin + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - name: Run optimized tests + run: tox -e optimized From 8eae0638976fa1968e3bbd364a49bb1eb689b655 Mon Sep 17 00:00:00 2001 From: Guruprasad Kamath Date: Fri, 18 Jul 2025 13:32:02 +0200 Subject: [PATCH 2/6] run on ubuntu --- .github/workflows/test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9e3c581b7f..45a3684eff 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,7 +11,7 @@ on: jobs: static: - runs-on: [self-hosted-ghr, size-xl-x64] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: @@ -38,7 +38,7 @@ jobs: run: tox -e static py3_eest: - runs-on: [self-hosted-ghr, size-xl-x64] + runs-on: ubuntu-latest needs: static steps: - uses: actions/checkout@v4 @@ -66,7 +66,7 @@ jobs: run: tox -e py3_eest pypy3: - runs-on: [self-hosted-ghr, size-xl-x64] + runs-on: ubuntu-latest needs: static steps: - uses: actions/checkout@v4 @@ -96,7 +96,7 @@ jobs: PYPY_GC_MAX: "10G" py3: - runs-on: [self-hosted-ghr, size-xl-x64] + runs-on: ubuntu-latest needs: [py3_eest, pypy3] steps: - uses: actions/checkout@v4 @@ -130,7 +130,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} optimized: - runs-on: [self-hosted-ghr, size-xl-x64] + runs-on: ubuntu-latest needs: [py3_eest, pypy3] steps: - uses: actions/checkout@v4 From 5d07bd9c129c5537ef1e7ea395b40bed6c8e2289 Mon Sep 17 00:00:00 2001 From: Guruprasad Kamath Date: Fri, 18 Jul 2025 13:35:24 +0200 Subject: [PATCH 3/6] sudo debian front-end --- .github/workflows/test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 45a3684eff..28f5488c66 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -27,7 +27,7 @@ jobs: python-version: "3.11" - name: Install Tox and any other packages run: | - DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config + sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config pip install 'tox>=4.11,<5' requests - name: Download Geth and add to $PATH run: | @@ -55,7 +55,7 @@ jobs: python-version: "3.11" - name: Install Tox and any other packages run: | - DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config + sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config pip install 'tox>=4.11,<5' requests - name: Download Geth and add to $PATH run: | @@ -83,7 +83,7 @@ jobs: python-version: "pypy3.11" - name: Install Tox and any other packages run: | - DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config + sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config pip install 'tox>=4.11,<5' requests - name: Download Geth and add to $PATH run: | @@ -113,7 +113,7 @@ jobs: python-version: "3.11" - name: Install Tox and any other packages run: | - DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config + sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config pip install 'tox>=4.11,<5' requests - name: Download Geth and add to $PATH run: | @@ -147,7 +147,7 @@ jobs: python-version: "3.11" - name: Install Tox and any other packages run: | - DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config + sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config pip install 'tox>=4.11,<5' requests - name: Download Geth and add to $PATH run: | From c1dbceaa761cb369dd207b765b0554ab9f42d792 Mon Sep 17 00:00:00 2001 From: Guruprasad Kamath Date: Fri, 18 Jul 2025 13:51:10 +0200 Subject: [PATCH 4/6] create composite action --- .github/actions/setup-env/action.yaml | 23 ++++++++ .github/workflows/test.yaml | 75 ++------------------------- 2 files changed, 28 insertions(+), 70 deletions(-) create mode 100644 .github/actions/setup-env/action.yaml diff --git a/.github/actions/setup-env/action.yaml b/.github/actions/setup-env/action.yaml new file mode 100644 index 0000000000..18d8f611b8 --- /dev/null +++ b/.github/actions/setup-env/action.yaml @@ -0,0 +1,23 @@ +name: Setup Environment +description: Common setup for Ethereum Spec jobs +runs: + using: "composite" + steps: + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: "1.69.0" + override: true + + - name: Install Tox and any other packages + shell: bash + run: | + sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config + pip install 'tox>=4.11,<5' requests + + - name: Download Geth and add to $PATH + shell: bash + run: | + mkdir -p $GITHUB_WORKSPACE/bin + $GITHUB_WORKSPACE/scripts/download_geth_linux.py --dir $GITHUB_WORKSPACE/bin + echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH \ No newline at end of file diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 28f5488c66..1da04d906e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,24 +16,11 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: "1.69.0" - override: true - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.11" - - name: Install Tox and any other packages - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config - pip install 'tox>=4.11,<5' requests - - name: Download Geth and add to $PATH - run: | - mkdir -p $GITHUB_WORKSPACE/bin - $GITHUB_WORKSPACE/scripts/download_geth_linux.py --dir $GITHUB_WORKSPACE/bin - echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - uses: ./.github/actions/setup-env - name: Run static checks run: tox -e static @@ -44,24 +31,11 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: "1.69.0" - override: true - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.11" - - name: Install Tox and any other packages - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config - pip install 'tox>=4.11,<5' requests - - name: Download Geth and add to $PATH - run: | - mkdir -p $GITHUB_WORKSPACE/bin - $GITHUB_WORKSPACE/scripts/download_geth_linux.py --dir $GITHUB_WORKSPACE/bin - echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - uses: ./.github/actions/setup-env - name: Run py3_eest tests run: tox -e py3_eest @@ -72,24 +46,11 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: "1.69.0" - override: true - name: Setup Python uses: actions/setup-python@v5 with: python-version: "pypy3.11" - - name: Install Tox and any other packages - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config - pip install 'tox>=4.11,<5' requests - - name: Download Geth and add to $PATH - run: | - mkdir -p $GITHUB_WORKSPACE/bin - $GITHUB_WORKSPACE/scripts/download_geth_linux.py --dir $GITHUB_WORKSPACE/bin - echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - uses: ./.github/actions/setup-env - name: Run pypy3 tests run: tox -e pypy3 env: @@ -102,24 +63,11 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: "1.69.0" - override: true - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.11" - - name: Install Tox and any other packages - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config - pip install 'tox>=4.11,<5' requests - - name: Download Geth and add to $PATH - run: | - mkdir -p $GITHUB_WORKSPACE/bin - $GITHUB_WORKSPACE/scripts/download_geth_linux.py --dir $GITHUB_WORKSPACE/bin - echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - uses: ./.github/actions/setup-env - name: Run py3 tests run: tox -e py3 - name: Upload coverage reports to Codecov @@ -136,23 +84,10 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: "1.69.0" - override: true - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.11" - - name: Install Tox and any other packages - run: | - sudo DEBIAN_FRONTEND=noninteractive apt-get install --yes --force-yes build-essential pkg-config - pip install 'tox>=4.11,<5' requests - - name: Download Geth and add to $PATH - run: | - mkdir -p $GITHUB_WORKSPACE/bin - $GITHUB_WORKSPACE/scripts/download_geth_linux.py --dir $GITHUB_WORKSPACE/bin - echo $GITHUB_WORKSPACE/bin >> $GITHUB_PATH + - uses: ./.github/actions/setup-env - name: Run optimized tests run: tox -e optimized From 1b4d805384df75ff81aac51fbe396052ae23f4bd Mon Sep 17 00:00:00 2001 From: Guruprasad Kamath Date: Fri, 18 Jul 2025 13:55:53 +0200 Subject: [PATCH 5/6] rename jobs --- .github/workflows/test.yaml | 16 ++++++++-------- tox.ini | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1da04d906e..81b00800a8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -24,7 +24,7 @@ jobs: - name: Run static checks run: tox -e static - py3_eest: + py3: runs-on: ubuntu-latest needs: static steps: @@ -36,8 +36,8 @@ jobs: with: python-version: "3.11" - uses: ./.github/actions/setup-env - - name: Run py3_eest tests - run: tox -e py3_eest + - name: Run py3 tests + run: tox -e py3 pypy3: runs-on: ubuntu-latest @@ -56,9 +56,9 @@ jobs: env: PYPY_GC_MAX: "10G" - py3: + json_infra: runs-on: ubuntu-latest - needs: [py3_eest, pypy3] + needs: [py3, pypy3] steps: - uses: actions/checkout@v4 with: @@ -68,8 +68,8 @@ jobs: with: python-version: "3.11" - uses: ./.github/actions/setup-env - - name: Run py3 tests - run: tox -e py3 + - name: Run json infra tests + run: tox -e json_infra - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5 with: @@ -79,7 +79,7 @@ jobs: optimized: runs-on: ubuntu-latest - needs: [py3_eest, pypy3] + needs: [py3, pypy3] steps: - uses: actions/checkout@v4 with: diff --git a/tox.ini b/tox.ini index 8ae7dd2a26..b75c374932 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ commands = ethereum-spec-lint vulture src tests vulture_whitelist.py --exclude "*/tests/fixtures/*" --ignore-names "pytest_*" -[testenv:py3] +[testenv:json_infra] extras = test commands = @@ -33,7 +33,7 @@ commands = --basetemp="{temp_dir}/pytest" \ tests -[testenv:py3_eest] +[testenv:py3] extras = test commands = From 5562ffd478cec4f2d9835d41e26038b1c90d4c82 Mon Sep 17 00:00:00 2001 From: Guruprasad Kamath Date: Fri, 18 Jul 2025 14:09:49 +0200 Subject: [PATCH 6/6] update dependencies --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 81b00800a8..7e9a174d10 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -58,7 +58,7 @@ jobs: json_infra: runs-on: ubuntu-latest - needs: [py3, pypy3] + needs: py3 steps: - uses: actions/checkout@v4 with: @@ -79,7 +79,7 @@ jobs: optimized: runs-on: ubuntu-latest - needs: [py3, pypy3] + needs: py3 steps: - uses: actions/checkout@v4 with: