From 1a021ea40529ca09ff4d87dd4dc97beddde860d1 Mon Sep 17 00:00:00 2001 From: Ibrahim Hadzic Date: Sat, 30 Nov 2024 00:56:56 -0500 Subject: [PATCH] Update github workflows --- .github/workflows/build.yml | 21 +-- ...o-publish.yml => version_and_coverage.yml} | 29 +++- Makefile | 130 +++++------------- diagram.svg | 1 - 4 files changed, 75 insertions(+), 106 deletions(-) rename .github/workflows/{auto-publish.yml => version_and_coverage.yml} (52%) delete mode 100644 diagram.svg diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 80a35c81..279ea660 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,12 +1,13 @@ -name: Lighter CI +name: CI on: push: - branches: - - main + branches: [main] pull_request: - branches: - - main + branches: [main] + +permissions: + contents: write jobs: build: @@ -16,9 +17,9 @@ jobs: python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2.2.2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -26,10 +27,10 @@ jobs: run: make setup - name: Set up cache - uses: actions/cache@v2.1.6 + uses: actions/cache@v3 with: path: .venv - key: venv-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }} + key: venv-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }} - name: Install dependencies run: | @@ -42,6 +43,7 @@ jobs: make check-safety - name: Run style checks + continue-on-error: true run: | poetry install --with style make check-codestyle @@ -50,4 +52,3 @@ jobs: run: | poetry install --with tests make test - diff --git a/.github/workflows/auto-publish.yml b/.github/workflows/version_and_coverage.yml similarity index 52% rename from .github/workflows/auto-publish.yml rename to .github/workflows/version_and_coverage.yml index 85862d70..d67d758e 100644 --- a/.github/workflows/auto-publish.yml +++ b/.github/workflows/version_and_coverage.yml @@ -1,5 +1,12 @@ -name: Auto Publish +# Workflow to handle version bumping and coverage badge updates +name: Version and Coverage + +# Trigger on CI workflow completion or direct pushes to main on: + workflow_run: + workflows: ["CI"] + types: + - completed push: branches: - main @@ -8,22 +15,37 @@ jobs: build: runs-on: ubuntu-latest steps: + # Set up Python environment - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2.2.2 with: - python-version: 3.9 + python-version: 3.11 + # Install Poetry package manager - name: Install poetry run: make setup + # Install poeblix for managing package versions in lockfile - name: Install poeblix for version freezing run: poetry self add poeblix@latest + # Bump the project version number - name: Bump version run: | poetry version prerelease + # Install coverage-badge tool for generating coverage SVG + - name: Install coverage-badge + run: | + poetry run pip install coverage-badge + + # Generate the coverage badge SVG + - name: Update the coverage badge + run: | + make coverage + + # Commit version bump and coverage badge changes - name: Commit changes run: | git config --local user.email "action@github.com" @@ -31,12 +53,15 @@ jobs: git add . git commit -m "Bump version" || echo "No changes to commit" + # Push changes back to the repository - name: Push changes run: git push + # Build the package using locked versions - name: Build with lockfile versions run: poetry blixbuild --only-lock + # Publish the package to PyPI using stored credentials - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/Makefile b/Makefile index 7ca712ff..00ebc43f 100644 --- a/Makefile +++ b/Makefile @@ -1,127 +1,71 @@ -#* Variables +### Variables +# Define shell and Python environment variables SHELL := /usr/bin/env bash PYTHON := python PYTHONPATH := `pwd` -#* Docker variables -IMAGE := lighter -VERSION := latest - -#* Poetry - -#* Fix for issue with poetry +### Poetry Setup +# Install Poetry and configure environment .PHONY: setup -setup: poetry-download - @echo export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring >> ~/.bashrc - @echo export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring >> ~/.profile - source ~/.bashrc - source ~/.profile - poetry self add poetry-bumpversion@latest - poetry self add poetry-plugin-export@latest - - -.PHONY: poetry-download -poetry-download: +setup: curl -sSL https://install.python-poetry.org | $(PYTHON) - + @echo "export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring" >> ~/.bashrc + @echo "export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring" >> ~/.profile + poetry self add poetry-bumpversion@latest poetry-plugin-export@latest -.PHONY: poetry-remove -poetry-remove: - curl -sSL https://install.python-poetry.org | $(PYTHON) - --uninstall - -#* Installation +### Installation +# Install project dependencies .PHONY: install install: poetry install -n +# Install pre-commit hooks .PHONY: pre-commit-install pre-commit-install: poetry run pre-commit install -#* Formatters -.PHONY: codestyle -codestyle: - poetry run pyupgrade --exit-zero-even-if-changed --py37-plus **/*.py - poetry run isort --settings-path pyproject.toml ./ - poetry run black --config pyproject.toml ./ - -.PHONY: formatting -formatting: codestyle - -#* Linting -.PHONY: test -test: - PYTHONPATH=$(PYTHONPATH) poetry run pytest -c pyproject.toml --cov-report=html --cov=lighter tests/ - poetry run coverage-badge -o assets/images/coverage.svg -f - +### Linting & Testing +# Check code formatting and style .PHONY: check-codestyle check-codestyle: poetry run isort --diff --check-only --settings-path pyproject.toml ./ poetry run black --diff --check --config pyproject.toml ./ poetry run pylint lighter -.PHONY: mypy -mypy: - poetry run mypy --config-file pyproject.toml ./ - +# Run security checks .PHONY: check-safety check-safety: poetry check poetry export | poetry run safety check --stdin poetry run bandit -ll --recursive lighter tests -.PHONY: lint -lint: test check-codestyle mypy check-safety +# Run tests with coverage report +.PHONY: test +test: + PYTHONPATH=$(PYTHONPATH) poetry run pytest -c pyproject.toml --cov-report=html --cov=lighter tests/ + +# Generate coverage badge +.PHONY: coverage +coverage: + poetry run coverage-badge -o assets/images/coverage.svg -f +### Dependency Management +# Update development dependencies .PHONY: update-dev-deps update-dev-deps: - poetry add -G dev bandit@latest "isort[colors]@latest" mypy@latest pre-commit@latest pydocstyle@latest pylint@latest pytest@latest pyupgrade@latest safety@latest coverage@latest coverage-badge@latest pytest-html@latest pytest-cov@latest + poetry add -G dev bandit@latest "isort[colors]@latest" mypy@latest pre-commit@latest pydocstyle@latest \ + pylint@latest pytest@latest pyupgrade@latest safety@latest coverage@latest coverage-badge@latest \ + pytest-html@latest pytest-cov@latest poetry add -G dev --allow-prereleases black@latest -#* Docker -# Example: make docker-build VERSION=latest -# Example: make docker-build IMAGE=some_name VERSION=0.0.1 -.PHONY: docker-build -docker-build: - @echo Building docker $(IMAGE):$(VERSION) ... - docker build \ - -t $(IMAGE):$(VERSION) . \ - -f ./docker/Dockerfile --no-cache - -# Example: make docker-remove VERSION=latest -# Example: make docker-remove IMAGE=some_name VERSION=0.0.1 -.PHONY: docker-remove -docker-remove: - @echo Removing docker $(IMAGE):$(VERSION) ... - docker rmi -f $(IMAGE):$(VERSION) - -#* Cleaning -.PHONY: pycache-remove -pycache-remove: - find . | grep -E "(__pycache__|\.pyc|\.pyo$$)" | xargs rm -rf +# Update main project dependencies +.PHONY: update-deps +update-deps: + poetry add torch@latest torchvision@latest pytorch_lightning@latest torchmetrics@latest monai@latest -.PHONY: dsstore-remove -dsstore-remove: - find . | grep -E ".DS_Store" | xargs rm -rf - -.PHONY: mypycache-remove -mypycache-remove: - find . | grep -E ".mypy_cache" | xargs rm -rf - -.PHONY: ipynbcheckpoints-remove -ipynbcheckpoints-remove: - find . | grep -E ".ipynb_checkpoints" | xargs rm -rf - -.PHONY: pytestcache-remove -pytestcache-remove: - find . | grep -E ".pytest_cache" | xargs rm -rf - -.PHONY: build-remove -build-remove: +### Cleaning +# Remove temporary files and build artifacts +.PHONY: clean +clean: + find . | grep -E "(__pycache__|\.pyc|\.pyo$|.DS_Store|.mypy_cache|.pytest_cache|.ipynb_checkpoints)" | xargs rm -rf rm -rf build/ - -.PHONY: cleanup -cleanup: pycache-remove dsstore-remove mypycache-remove ipynbcheckpoints-remove pytestcache-remove - -.PHONY: get-latest-deps -get-latest-deps: - poetry add torch@latest torchvision@latest pytorch_lightning@latest torchmetrics@latest monai@latest \ No newline at end of file diff --git a/diagram.svg b/diagram.svg deleted file mode 100644 index bf43c409..00000000 --- a/diagram.svg +++ /dev/null @@ -1 +0,0 @@ -teststestsprojectsprojectslighterlighterdockerdockerintegrationintegrationcifar10cifar10utilsutilscallbackscallbacksmodelsmodelswriterwriterunit/test_du...unit/test_du...unit/test_du...system.pysystem.pysystem.pycontribcontribcontribpoetry.lockpoetry.lockpoetry.lock.safety-poli....safety-poli....safety-poli...pyproject.tomlpyproject.tomlpyproject.tomlCODE_OF_CON...CODE_OF_CON...CODE_OF_CON...MakefileMakefileMakefile.gitignore.gitignore.gitignoreexperiments/...experiments/...experiments/...model.pymodel.pymodel.pyfreezer.pyfreezer.pyfreezer.pycollate.pycollate.pycollate.pyrunner.pyrunner.pyrunner.pymisc.pymisc.pymisc.pydynamic_...dynamic_...dynamic_...logger.pylogger.pylogger.pyutils.pyutils.pyutils.pybase.pybase.pybase.pyfile.pyfile.pyfile.pytable.pytable.pytable.py.gitignore.md.py.svg.toml.yaml.ymleach dot sized by file size \ No newline at end of file