Merge pull request #96 from grok-ai/feature/hf-integration #291
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Suite | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - develop | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| env: | |
| CACHE_NUMBER: 4 # increase to reset cache manually | |
| CONDA_ENV_FILE: 'env.yaml' | |
| CONDA_ENV_NAME: 'project-test' | |
| COOKIECUTTER_PROJECT_NAME: 'project-test' | |
| HUGGING_FACE_HUB_TOKEN: ${{secrets.HUGGING_FACE_HUB_TOKEN}} | |
| jobs: | |
| build: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.11'] | |
| include: | |
| - os: ubuntu-20.04 | |
| label: linux-64 | |
| prefix: /usr/share/miniconda3/envs/ | |
| # - os: macos-latest | |
| # label: osx-64 | |
| # prefix: /Users/runner/miniconda3/envs/$CONDA_ENV_NAME | |
| # - os: windows-latest | |
| # label: win-64 | |
| # prefix: C:\Miniconda3\envs\$CONDA_ENV_NAME | |
| name: ${{ matrix.label }}-py${{ matrix.python-version }} | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Parametrize conda env name | |
| run: echo "PY_CONDA_ENV_NAME=${{ env.CONDA_ENV_NAME }}-${{ matrix.python-version }}" >> $GITHUB_ENV | |
| - name: echo conda env name | |
| run: echo ${{ env.PY_CONDA_ENV_NAME }} | |
| - name: Parametrize conda prefix | |
| run: echo "PY_PREFIX=${{ matrix.prefix }}${{ env.PY_CONDA_ENV_NAME }}" >> $GITHUB_ENV | |
| - name: echo conda prefix | |
| run: echo ${{ env.PY_PREFIX }} | |
| - name: Define generated project files paths | |
| run: | | |
| echo "PROJECT_SETUPCFG_FILE=${{ env.COOKIECUTTER_PROJECT_NAME }}/setup.cfg" >> $GITHUB_ENV | |
| echo "PROJECT_CONDAENV_FILE=${{ env.COOKIECUTTER_PROJECT_NAME }}/${{ env.CONDA_ENV_FILE }}" >> $GITHUB_ENV | |
| echo "PROJECT_PRECOMMIT_FILE=${{ env.COOKIECUTTER_PROJECT_NAME }}/.pre-commit-config.yaml" >> $GITHUB_ENV | |
| - uses: actions/checkout@v2 | |
| # COOKIECUTTER GENERATION | |
| - name: Set up Python | |
| uses: actions/setup-python@v1 | |
| - name: Install Dependencies | |
| shell: bash -l {0} | |
| run: | | |
| pip install cookiecutter | |
| - name: Generate Repo | |
| shell: bash -l {0} | |
| run: | | |
| echo -e 'n\nn\nn\n' | cookiecutter . --no-input project_name=${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| - name: Init git into generated repo | |
| shell: bash -l {0} | |
| run: | | |
| git config --global user.name ci-bot | |
| git config --global user.email [email protected] | |
| git init | |
| git add --all | |
| git commit -m "Initial commit" | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| - name: Define cache key postfix | |
| run: | | |
| echo "CACHE_KEY_POSTFIX=${{ matrix.label }}-${{ matrix.python-version }}-${{ env.CACHE_NUMBER }}-${{ env.PY_CONDA_ENV_NAME }}-${{ hashFiles(env.PROJECT_CONDAENV_FILE) }}-${{ hashFiles(env.PROJECT_SETUPCFG_FILE) }}" >> $GITHUB_ENV | |
| - name: Echo cache keys | |
| run: | | |
| echo ${{ env.PROJECT_SETUPCFG_FILE }} | |
| echo ${{ env.PROJECT_CONDAENV_FILE }} | |
| echo ${{ env.PROJECT_PRECOMMIT_FILE }} | |
| echo ${{ env.CACHE_KEY_POSTFIX }} | |
| # GENERATED PROJECT CI/CD | |
| # Remove the python version pin from the env.yml which could be inconsistent | |
| - name: Remove explicit python version from the environment | |
| shell: bash -l {0} | |
| run: | | |
| sed -Ei '/^\s*-?\s*python\s*([#=].*)?$/d' ${{ env.CONDA_ENV_FILE }} | |
| cat ${{ env.CONDA_ENV_FILE }} | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| # Install torch cpu-only | |
| - name: Install torch cpu only | |
| shell: bash -l {0} | |
| run: | | |
| sed -i '/nvidia\|cuda/d' ${{ env.CONDA_ENV_FILE }} | |
| cat ${{ env.CONDA_ENV_FILE }} | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| - name: Setup Mambaforge | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| miniforge-variant: Mambaforge | |
| miniforge-version: latest | |
| activate-environment: ${{ env.PY_CONDA_ENV_NAME }} | |
| python-version: ${{ matrix.python-version }} | |
| use-mamba: true | |
| - uses: actions/cache@v2 | |
| name: Conda cache | |
| with: | |
| path: ${{ env.PY_PREFIX }} | |
| key: conda-${{ env.CACHE_KEY_POSTFIX }} | |
| id: conda_cache | |
| - uses: actions/cache@v2 | |
| name: Pip cache | |
| with: | |
| path: ~/.cache/pip | |
| key: pip-${{ env.CACHE_KEY_POSTFIX }} | |
| - uses: actions/cache@v2 | |
| name: Pre-commit cache | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: pre-commit-${{ hashFiles(env.PROJECT_PRECOMMIT_FILE) }}-conda-${{ env.CACHE_KEY_POSTFIX }} | |
| # Ensure the hack for the python version worked | |
| - name: Ensure we have the right Python | |
| shell: bash -l {0} | |
| run: | | |
| echo "Installed Python: $(python --version)" | |
| echo "Expected: ${{ matrix.python-version }}" | |
| python --version | grep "Python ${{ matrix.python-version }}" | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| # https://stackoverflow.com/questions/70520120/attributeerror-module-setuptools-distutils-has-no-attribute-version | |
| # https://github.com/pytorch/pytorch/pull/69904 | |
| - name: Downgrade setuptools due to a but in PyTorch 1.10.1 | |
| shell: bash -l {0} | |
| run: | | |
| pip install setuptools==59.5.0 --upgrade | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| - name: Update conda environment | |
| run: mamba env update -n ${{ env.PY_CONDA_ENV_NAME }} -f ${{ env.CONDA_ENV_FILE }} | |
| if: steps.conda_cache.outputs.cache-hit != 'true' | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| # Update pip env whether or not there was a conda cache hit | |
| - name: Update pip environment | |
| shell: bash -l {0} | |
| run: pip install -e ".[dev]" | |
| if: steps.conda_cache.outputs.cache-hit == 'true' | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| - run: pip3 list | |
| shell: bash -l {0} | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| - run: mamba info | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| - run: mamba list | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| # Ensure the hack for the python version worked | |
| - name: Ensure we have the right Python | |
| shell: bash -l {0} | |
| run: | | |
| echo "Installed Python: $(python --version)" | |
| echo "Expected: ${{ matrix.python-version }}" | |
| python --version | grep "Python ${{ matrix.python-version }}" | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| - name: Run pre-commits | |
| shell: bash -l {0} | |
| run: | | |
| pre-commit install | |
| pre-commit run -v --all-files --show-diff-on-failure | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} | |
| - name: Test with pytest | |
| shell: bash -l {0} | |
| run: | | |
| pytest -v | |
| working-directory: ${{ env.COOKIECUTTER_PROJECT_NAME }} |