refactor: Refactored setup and tool config into setup.cfg (#57)

* refactor: Moved all config to setup.cfg

* chore: Updated gitignore

* style: Removed unused import

* refactor: Updated requirement validation script

* ci: Fixed CI test extra installation

Author: frgfm

.coveragerc

@@ -3,60 +3,88 @@
 import requirements
 from requirements.requirement import Requirement
 
-# Deps that won't have a specific requirements.txt
-IGNORE = ["flake8", "isort", "mypy", "pydocstyle"]
 # All req files to check
-REQ_FILES = ["requirements.txt", "tests/requirements.txt", "docs/requirements.txt"]
+EXTRA_MAP = {
+    "test": "tests/requirements.txt",
+    "docs": "docs/requirements.txt",
+}
+EXTRA_IGNORE = ["dev"]
 
 
-def main():
+def parse_deps(deps):
+    reqs = {}
+    for _dep in deps:
+        lib, specs = _dep
+        assert reqs.get(lib) is None, f"conflicting deps for {lib}"
+        reqs[lib] = specs
 
-    # Collect the deps from all requirements.txt
-    folder = Path(__file__).parent.parent.absolute()
-    req_deps = {}
-    for file in REQ_FILES:
-        with open(folder.joinpath(file), 'r') as f:
-            _deps = [(req.name, req.specs) for req in requirements.parse(f)]
+    return reqs
 
-        for _dep in _deps:
-            lib, specs = _dep
-            assert req_deps.get(lib, specs) == specs, f"conflicting deps for {lib}"
-            req_deps[lib] = specs
 
-    # Collect the one from setup.py
-    setup_deps = {}
-    with open(folder.joinpath("setup.py"), 'r') as f:
-        setup = f.readlines()
-    lines = setup[setup.index("_deps = [\n") + 1:]
-    lines = [_dep.strip() for _dep in lines[:lines.index("]\n")]]
-    lines = [_dep.split('"')[1] for _dep in lines if _dep.startswith('"')]
-    _reqs = [Requirement.parse(_line) for _line in lines]
-    _deps = [(req.name, req.specs) for req in _reqs]
-    for _dep in _deps:
-        lib, specs = _dep
-        assert setup_deps.get(lib) is None, f"conflicting deps for {lib}"
-        setup_deps[lib] = specs
+def get_conficts(setup_reqs, requirement_file):
+
+    # Parse the deps from the requirements.txt
+    folder = Path(__file__).parent.parent.absolute()
+    req_deps = {}
+    with open(folder.joinpath(requirement_file), 'r') as f:
+        _deps = [(req.name, req.specs) for req in requirements.parse(f)]
 
-    # Remove ignores
-    for k in IGNORE:
-        if isinstance(req_deps.get(k), list):
-            del req_deps[k]
-        if isinstance(setup_deps.get(k), list):
-            del setup_deps[k]
+    req_deps = parse_deps(_deps)
 
     # Compare them
-    assert len(req_deps) == len(setup_deps)
+    assert len(req_deps) == len(setup_reqs)
     mismatches = []
-    for k, v in setup_deps.items():
+    for k, v in setup_reqs.items():
         assert isinstance(req_deps.get(k), list)
         if req_deps[k] != v:
             mismatches.append((k, v, req_deps[k]))
 
-    if len(mismatches) > 0:
+    return mismatches
+
+
+def main():
+
+    # Collect the one from setup.py
+    folder = Path(__file__).parent.parent.absolute()
+    with open(folder.joinpath("setup.cfg"), 'r') as f:
+        setup = f.readlines()
+
+    # install_requires
+    lines = setup[setup.index("install_requires =\n") + 1:]
+    lines = [_dep.strip() for _dep in lines[:lines.index("\n")]]
+    _reqs = [Requirement.parse(_line) for _line in lines]
+    install_requires = parse_deps([(req.name, req.specs) for req in _reqs])
+
+    # extras
+    extras_require = {}
+    lines = setup[setup.index("[options.extras_require]\n") + 1:]
+    lines = lines[:lines.index("\n")]
+    # Split each extra
+    extra_lines = [_line for _line in lines if str.isalpha(_line[0])]
+    extra_names = [_line.strip().replace("=", "").strip() for _line in extra_lines]
+    for current_extra, start_line, end_line in zip(extra_names, extra_lines, extra_lines[1:] + [None]):
+        if current_extra in EXTRA_IGNORE:
+            continue
+        _lines = [_dep for _dep in lines[lines.index(start_line) + 1:]]
+        if isinstance(end_line, str):
+            _lines = _lines[:_lines.index(end_line)]
+        # Remove comments
+        _lines = [_line.strip() for _line in _lines]
+        _reqs = [Requirement.parse(_line.strip()) for _line in _lines if not _line.strip().startswith("#")]
+        extras_require[current_extra] = parse_deps([(req.name, req.specs) for req in _reqs])
+
+    # Resolve conflicts
+    mismatches = {}
+    mismatches["requirements.txt"] = get_conficts(install_requires, "requirements.txt")
+    for extra_k, req_file in EXTRA_MAP.items():
+        mismatches[req_file] = get_conficts(extras_require[extra_k], req_file)
+
+    # Display the results
+    if any(len(mismatch) > 0 for mismatch in mismatches.values()):
         mismatch_str = "version specifiers mismatches:\n"
         mismatch_str += '\n'.join(
-            f"- {lib}: {setup} (from setup.py) | {reqs} (from requirements)"
-            for lib, setup, reqs in mismatches
+            f"- {lib}: {setup} (from setup.cfg) | {reqs} (from {req_file})"
+            for req_file, issues in mismatches.items() for lib, setup, reqs in issues
         )
         raise AssertionError(mismatch_str)
 

.flake8

@@ -28,9 +28,6 @@ jobs:
           key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('**/*.py') }}
           restore-keys: |
             ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-
-            ${{ runner.os }}-pkg-deps-
-            ${{ runner.os }}-
       - name: Install package
         run: |
           python -m pip install --upgrade pip

.github/validate_deps.py

@@ -27,9 +27,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('docs/requirements.txt') }}-
             ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-
-            ${{ runner.os }}-pkg-deps-
-            ${{ runner.os }}-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/builds.yml

@@ -7,38 +7,7 @@ on:
     branches: master
 
 jobs:
-  install:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        python: [3.7]
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          persist-credentials: false
-      - name: Set up Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python }}
-          architecture: x64
-      - name: Cache python modules
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('**/*.py') }}
-          restore-keys: |
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-
-            ${{ runner.os }}-pkg-deps-
-            ${{ runner.os }}-
-      - name: Install package
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e . --upgrade
-
   pytest:
-    needs: install
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -61,13 +30,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('tests/requirements.txt') }}-
             ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-
-            ${{ runner.os }}-pkg-deps-
-            ${{ runner.os }}-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -e ".[testing]" --upgrade
+          pip install -e ".[test]" --upgrade
       - name: Run unittests
         run: |
           coverage run -m pytest tests/
@@ -91,7 +57,6 @@ jobs:
 
   docs-build:
     runs-on: ${{ matrix.os }}
-    needs: install
     strategy:
       matrix:
         os: [ubuntu-latest]
@@ -113,9 +78,6 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('docs/requirements.txt') }}-
             ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-
-            ${{ runner.os }}-pkg-deps-
-            ${{ runner.os }}-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/docs.yml

@@ -21,8 +21,6 @@ jobs:
           key: ${{ runner.os }}-pkg-deps-${{ hashFiles('requirements.txt') }}-${{ hashFiles('**/*.py') }}
           restore-keys: |
             ${{ runner.os }}-pkg-deps-${{ hashFiles('requirements.txt') }}-
-            ${{ runner.os }}-pkg-deps-
-            ${{ runner.os }}-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/main.yml

@@ -66,9 +66,6 @@ jobs:
           key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('**/*.py') }}
           restore-keys: |
             ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
-            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-
-            ${{ runner.os }}-pkg-deps-
-            ${{ runner.os }}-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/release.yml

@@ -130,6 +130,6 @@ dmypy.json
 .pyre/
 
 # Package version
-/torchscan/version.py
+torchscan/version.py
 # Conda distribution
 conda-dist/

.github/workflows/style.yml

@@ -1,3 +1,90 @@
 [metadata]
-description-file = README.md
-license_file = LICENSE
+author = François-Guillaume Fernandez
+author_email = [email protected]
+description = Useful information about your Pytorch module
+long_description = file: README.md
+long_description_content_type = text/markdown; charset=UTF-8
+url = https://github.com/frgfm/torch-scan
+download_url = https://github.com/frgfm/torch-scan/tags
+project_urls = 
+    Documentation = https://frgfm.github.io/torch-scan
+    Source = https://github.com/frgfm/torch-scan
+    Tracker = https://github.com/frgfm/torch-scan/issues
+license = Apache
+license_file = LICENSE
+keywords = pytorch, deep learning, summary, memory, ram
+classifiers = 
+	Development Status :: 4 - Beta
+    Intended Audience :: Developers
+    Intended Audience :: Science/Research
+    License :: OSI Approved :: Apache Software License
+    Natural Language :: English
+    Operating System :: OS Independent
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Topic :: Scientific/Engineering
+    Topic :: Scientific/Engineering :: Mathematics
+    Topic :: Scientific/Engineering :: Artificial Intelligence
+
+[options]
+zip_safe = True
+packages = find:
+include_package_data = True
+python_requires = >=3.6.0,<4
+install_requires =
+	torch>=1.5.0
+
+[options.package_data]
+* = LICENSE
+
+[options.packages.find]
+exclude =
+    tests*
+
+[options.extras_require]
+test =
+    pytest>=5.3.2
+    coverage>=4.5.4
+quality =
+	flake8>=3.9.0
+    isort>=5.7.0
+    mypy>=0.812
+    pydocstyle>=6.0.0
+docs =
+	sphinx<=3.4.3
+    sphinx-rtd-theme==0.4.3
+    sphinxemoji>=0.1.8
+    sphinx-copybutton>=0.3.1
+    docutils<0.18
+    # cf. https://github.com/readthedocs/readthedocs.org/issues/9038
+    Jinja2<3.1
+dev =
+    %(test)s
+    %(quality)s
+    %(docs)s
+
+[flake8]
+max-line-length = 120
+ignore = E402, E265, F403, W503, W504, E731
+exclude = .github, .git, venv*, docs, build
+per-file-ignores = **/__init__.py:F401
+
+[mypy]
+files = torchscan/*.py
+show_error_codes = True
+pretty = True
+
+[isort]
+line_length = 120
+src_paths = torchscan, tests
+skip_glob = **/__init__.py
+known_third_party = torch, torchvision
+
+[pydocstyle]
+select = D300,D301,D417
+match = .*\.py
+
+[coverage:run]
+source = torchscan