ASV Benchmarks #245
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ASV Benchmarks | |
| on: | |
| push: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: "0 5 * * 4" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read # to fetch code (actions/checkout) | |
| env: | |
| ASV_CONFIG: asv.conf.json | |
| NAME_PREFIX: gha | |
| jobs: | |
| trigger: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # To fetch all commits to be able to generate benchmarks html | |
| token: ${{ secrets.AWS_BENCHMARKS }} | |
| - name: Prepare the tracker branch | |
| run: | | |
| git config --global user.email "project@openblas" | |
| git config --global user.name "OB benchmark bot" | |
| # XXX this assumes the `tracker` branch is the squash-merge from main | |
| # revert the past merge and merge the current main instead | |
| # (cannot just merge main without a revert because force-pushes to main) | |
| git checkout tracker | |
| git revert HEAD --no-edit | |
| # XXX needed locally not on CI? | |
| # git commit -am'revert squash-merging main' | |
| git merge main --squash | |
| git commit -am"squash-merge main at `git rev-parse main`" | |
| git push origin HEAD | |
| git checkout main | |
| bench: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # define matrix.name to identify github actions machine as hostname changes everytime | |
| - image: "cirun-aws-runner-graviton--${{ github.run_id }}" | |
| name: "gha-aws-graviton" | |
| - image: "cirun-aws-runner-cascade-lake--${{ github.run_id }}" | |
| name: "gha-aws-skylake" | |
| - image: ubuntu-24.04 | |
| name: "gha-ubuntu-noble" | |
| runs-on: ${{ matrix.image }} | |
| needs: trigger | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # To fetch all commits to be able to generate benchmarks html | |
| - name: Print system information | |
| run: | | |
| if [ "$RUNNER_OS" == "Linux" ]; then | |
| cat /proc/cpuinfo | |
| else | |
| echo "::error::$RUNNER_OS not supported" | |
| exit 1 | |
| fi | |
| - name: Install system dependencies | |
| run: | | |
| if [ "$RUNNER_OS" == "Linux" ]; then | |
| sudo apt update | |
| sudo apt-get install -y gfortran cmake ccache python3-pip pkg-config | |
| else | |
| echo "::error::$RUNNER_OS not supported" | |
| exit 1 | |
| fi | |
| - name: Install python dependencies | |
| run: | | |
| # --break-system-packages is required on ubuntu noble | |
| pip3 install "numpy<2" meson meson-python ninja build asv virtualenv --break-system-packages | |
| # install the nightly OpenBLAS wheel | |
| pip3 install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple scipy-openblas32 --break-system-packages | |
| # dump the pkg-config for the ASV build to pick up (cf $PKG_CONFIG_PATH usage under `Run benchmarks`) | |
| python3 -c'import scipy_openblas32 as so; print(so.get_pkg_config())' > scipy_openblas.pc | |
| - name: Print OpenBLAS information | |
| run: | | |
| echo "scipy_openblas.pc contents: " | |
| cat scipy_openblas.pc | |
| # store the OpenBLAS wheel info to add to the gh-pages commit message | |
| echo ${{ matrix.name }}":" > wheel_info | |
| python3 -c'import scipy_openblas32 as sc; print(f"version={sc.__version__} - {sc.get_openblas_config()}")' >> wheel_info | |
| - name: Set and log asv machine configuration | |
| run: | | |
| python3 -m asv machine --yes --config asv.conf.json | |
| echo "Machine Configuration:" | |
| cat ~/.asv-machine.json | |
| rm ~/.asv-machine.json | |
| # set the machine name depending on the OS/arch image | |
| echo "Setting machine name to ${{ matrix.name }}" | |
| python3 -m asv machine --machine ${{ matrix.name }} --yes --config $ASV_CONFIG -v | |
| cat ~/.asv-machine.json | |
| - name: Run benchmarks | |
| run: | | |
| # asv fails to checkout tracker unless we help it a bit: | |
| git checkout tracker | |
| git checkout main | |
| git branch | |
| python3 -m asv run --config $ASV_CONFIG -v | |
| env: | |
| PKG_CONFIG_PATH: ${{ github.workspace }} | |
| - name: Store/Upload benchmark results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.name }} | |
| path: | | |
| .asv/results/* | |
| wheel_info | |
| if-no-files-found: error | |
| combine-and-publish: | |
| runs-on: ubuntu-latest | |
| needs: bench | |
| steps: | |
| - name: Check out the repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # To fetch all commits to be able to generate benchmarks html | |
| token: ${{ secrets.AWS_BENCHMARKS }} | |
| - name: Download all artifacts from benchmarking runs | |
| uses: actions/download-artifact@v4 | |
| - name: Collect past runs | |
| run: | | |
| git checkout gh-pages | |
| mkdir -p .asv/results | |
| cp -r results .asv | |
| - name: Combine the runs | |
| run: | | |
| # NB artifact names start with gha- | |
| artifacts=`ls -d ./$NAME_PREFIX-*` | |
| echo "found ARTIFACTS = "$artifacts | |
| # put the benchmark results to where asv expects them: | |
| # .asv/ | |
| # results/ | |
| # benchmarks.json | |
| # machine-name-1/ | |
| # machine.json | |
| # commit_hash-1...json | |
| # machine-name-2/ | |
| # machine.json | |
| # commit_hash-1...json | |
| for dir in $artifacts; do cp -r $dir/.asv/* .asv; done | |
| # combine the wheel_info files from the artifacts | |
| for dir in $artifacts; do cat $dir/wheel_info >>./wheel_info; done | |
| cat wheel_info | |
| # return to main to be able to generate the new html report | |
| git checkout main | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Set up ASV and generate the html | |
| run: | | |
| git checkout tracker | |
| git checkout main | |
| pip install asv | |
| asv machine --yes --config $ASV_CONFIG | |
| asv publish --config $ASV_CONFIG -v | |
| - name: Upload the results | |
| run: | | |
| git config --global user.email "project@openblas" | |
| git config --global user.name "OB benchmark bot" | |
| asv gh-pages | |
| # persist the results/ directory with historic results | |
| git checkout gh-pages | |
| cp -r .asv/results results | |
| git add results/ | |
| git commit -am"add results for `git rev-parse origin/gh-pages`" -m"`cat wheel_info`" | |
| git push origin HEAD |