Merge pull request #209 from sarugaku/bugfix/204

Fix binary operator mapping in setup.py ast parser

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/ambv/black
-    rev: 19.10b0
+    rev: stable
     hooks:
       - id: black
         args: ["--target-version=py27", "--target-version=py37"]
@@ -11,11 +11,18 @@ repos:
       #- id: flake8
 
   - repo: https://github.com/asottile/seed-isort-config
-    rev: v1.9.3
+    rev: v2.1.0
     hooks:
       - id: seed-isort-config
+        args: [
+          --application-directories=src/requirementslib,
+          --settings-path=./,
+          # --exclude=tests/.*\.py,
+          # --exclude=src/requirementslib/models/setup_info.py
+        ]
 
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v4.3.21
+  - repo: https://github.com/timothycrosley/isort
+    rev: 4.3.21
     hooks:
       - id: isort
+

azure-pipelines.yml

@@ -43,7 +43,7 @@ jobs:
 
 - job: TestWindows
   pool:
-    vmImage: windows-2019
+    vmImage: windows-latest
   strategy:
     matrix:
       Python27:
@@ -54,13 +54,15 @@ jobs:
         python.version: '3.6'
       Python37:
         python.version: '3.7'
+      Python38:
+        python.version: '3.8'
     maxParallel: 4
   steps:
   - template: .azure-pipelines/templates/run-tests.yml
 
 - job: TestMacOS
   pool:
-    vmImage: macOS-10.13
+    vmImage: macOS-latest
   strategy:
     matrix:
       Python27:
@@ -71,6 +73,8 @@ jobs:
         python.version: '3.6'
       Python37:
         python.version: '3.7'
+      Python38:
+        python.version: '3.8'
     maxParallel: 4
   steps:
   - template: .azure-pipelines/templates/run-tests.yml

news/204.bugfix.rst

@@ -0,0 +1 @@
+Fixed an issue in binary operator mapping in the ``ast_parse_setup_py`` functionality of the dependency parser which could cause dependency resolution to fail.

news/206.bugfix.rst

@@ -0,0 +1 @@
+Fixed an issue which caused mappings of binary operators to fail to evaluate when parsing ``setup.py`` files.

news/207.bugfix.rst

@@ -0,0 +1 @@
+Fixed mapping and evaluation of boolean operators and comparisons when evaluating ``setup.py`` files with AST parser to discover dependencies.

setup.cfg

@@ -50,18 +50,17 @@ install_requires =
     attrs>=18.2
     cached_property
     distlib>=0.2.8
-    first
     orderedmultidict
     packaging>=19.0
     pep517>=0.5.0
     pip-shims>=0.3.2
     plette[validation]
+    python-dateutil
     requests
     scandir;python_version<"3.5"
     setuptools>=40.8
     six>=1.11.0
     tomlkit>=0.5.3
-    typing;python_version<"3.5"
     vistir>=0.3.1
 
 [options.extras_require]
@@ -89,6 +88,7 @@ tests =
     coverage
     hypothesis
 typing =
+    typing;python_version<"3.5"
     mypy;python_version>="3.4"
     mypy-extensions;python_version>="3.4"
     mypytools;python_version>="3.4"
@@ -102,8 +102,8 @@ universal = 1
 
 [tool:pytest]
 strict = true
-plugins = flake8 cov timeout
-addopts = -ra --cov -n auto --timeout 300
+plugins = flake8 cov xdist timeout
+addopts = -ra --cov=requirementslib -n auto --timeout 300
 testpaths = tests/
 norecursedirs = .* build dist news tasks docs tests/artifacts
 flake8-ignore =
@@ -127,7 +127,7 @@ not_skip = __init__.py
 line_length = 90
 indent = '    '
 multi_line_output = 3
-known_third_party = appdirs,attr,cached_property,distlib,environ,first,hypothesis,invoke,orderedmultidict,packaging,parver,pep517,pip_shims,pkg_resources,plette,pyparsing,pytest,requests,setuptools,six,tomlkit,towncrier,urllib3,vistir
+known_third_party = appdirs,attr,cached_property,distlib,environ,hypothesis,invoke,orderedmultidict,packaging,parver,pep517,pip_shims,pkg_resources,plette,pyparsing,pytest,requests,setuptools,six,tomlkit,towncrier,urllib3,vistir
 known_first_party = requirementslib,tests
 combine_as_imports=True
 include_trailing_comma = True

src/requirementslib/models/dependencies.py

@@ -11,7 +11,6 @@
 import packaging.version
 import pip_shims.shims
 import requests
-from first import first
 from packaging.utils import canonicalize_name
 from vistir.compat import JSONDecodeError, fs_str
 from vistir.contextmanagers import cd, temp_environ
@@ -20,6 +19,7 @@
 from ..environment import MYPY_RUNNING
 from ..utils import _ensure_dir, prepare_pip_source_args
 from .cache import CACHE_DIR, DependencyCache
+from .setup_info import SetupInfo
 from .utils import (
     clean_requires_python,
     fix_requires_python_marker,
@@ -144,9 +144,9 @@ def compatible_versions(self, other):
         :rtype: set(str)
         """
 
-        if len(self.candidates) == 1 and first(self.candidates).editable:
+        if len(self.candidates) == 1 and next(iter(self.candidates)).editable:
             return self
-        elif len(other.candidates) == 1 and first(other.candidates).editable:
+        elif len(other.candidates) == 1 and next(iter(other.candidates)).editable:
             return other
         return self.version_set & other.version_set
 
@@ -163,9 +163,9 @@ def compatible_abstract_dep(self, other):
 
         from .requirements import Requirement
 
-        if len(self.candidates) == 1 and first(self.candidates).editable:
+        if len(self.candidates) == 1 and next(iter(self.candidates)).editable:
             return self
-        elif len(other.candidates) == 1 and first(other.candidates).editable:
+        elif len(other.candidates) == 1 and next(iter(other.candidates)).editable:
             return other
         new_specifiers = self.specifiers & other.specifiers
         markers = set(self.markers) if self.markers else set()
@@ -480,90 +480,19 @@ def get_dependencies_from_index(dep, sources=None, pip_options=None, wheel_cache
     if not wheel_cache:
         wheel_cache = WHEEL_CACHE
     dep.is_direct = True
-    reqset = pip_shims.shims.RequirementSet()
-    reqset.add_requirement(dep)
     requirements = None
     setup_requires = {}
-    with temp_environ(), start_resolver(
-        finder=finder, session=session, wheel_cache=wheel_cache
-    ) as resolver:
+    with temp_environ():
         os.environ["PIP_EXISTS_ACTION"] = "i"
-        dist = None
         if dep.editable and not dep.prepared and not dep.req:
-            with cd(dep.setup_py_dir):
-                from setuptools.dist import distutils
-
-                try:
-                    dist = distutils.core.run_setup(dep.setup_py)
-                except (ImportError, TypeError, AttributeError):
-                    dist = None
-                else:
-                    setup_requires[dist.get_name()] = dist.setup_requires
-                if not dist:
-                    try:
-                        dist = dep.get_dist()
-                    except (TypeError, ValueError, AttributeError):
-                        pass
-                    else:
-                        setup_requires[dist.get_name()] = dist.setup_requires
-        resolver.require_hashes = False
-        try:
-            results = resolver._resolve_one(reqset, dep)
-        except Exception:
-            # FIXME: Needs to bubble the exception somehow to the user.
-            results = []
-        finally:
-            try:
-                wheel_cache.cleanup()
-            except AttributeError:
-                pass
-        resolver_requires_python = getattr(resolver, "requires_python", None)
-        requires_python = getattr(reqset, "requires_python", resolver_requires_python)
-        if requires_python:
-            add_marker = fix_requires_python_marker(requires_python)
-            reqset.remove(dep)
-            if dep.req.marker:
-                dep.req.marker._markers.extend(["and"].extend(add_marker._markers))
-            else:
-                dep.req.marker = add_marker
-            reqset.add(dep)
-        requirements = set()
-        for r in results:
-            if requires_python:
-                if r.req.marker:
-                    r.req.marker._markers.extend(["and"].extend(add_marker._markers))
-                else:
-                    r.req.marker = add_marker
-            requirements.add(format_requirement(r))
-        for section in setup_requires:
-            python_version = section
-            not_python = not is_python(section)
-
-            # This is for cleaning up :extras: formatted markers
-            # by adding them to the results of the resolver
-            # since any such extra would have been returned as a result anyway
-            for value in setup_requires[section]:
-
-                # This is a marker.
-                if is_python(section):
-                    python_version = value[1:-1]
-                else:
-                    not_python = True
-
-                if ":" not in value and not_python:
-                    try:
-                        requirement_str = "{0}{1}".format(value, python_version).replace(
-                            ":", ";"
-                        )
-                        requirements.add(
-                            format_requirement(
-                                make_install_requirement(requirement_str).ireq
-                            )
-                        )
-                    # Anything could go wrong here -- can't be too careful.
-                    except Exception:
-                        pass
-
+            setup_info = SetupInfo.from_ireq(dep)
+            results = setup_info.get_info()
+            setup_requires.update(results["setup_requires"])
+            requirements = set(results["requires"].values())
+        else:
+            results = pip_shims.shims.resolve(dep)
+            requirements = [v for v in results.values() if v.name != dep.name]
+        requirements = set([format_requirement(r) for r in requirements])
     if not dep.editable and is_pinned_requirement(dep) and requirements is not None:
         DEPENDENCY_CACHE[dep] = list(requirements)
     return requirements
@@ -694,10 +623,10 @@ def get_grouped_dependencies(constraints):
     # then we take the loose match (which _is_ flexible) and start moving backwards in
     # versions by popping them off of a stack and checking for the conflicting package
     for _, ireqs in full_groupby(constraints, key=key_from_ireq):
-        ireqs = list(ireqs)
-        editable_ireq = first(ireqs, key=lambda ireq: ireq.editable)
+        ireqs = sorted(ireqs, key=lambda ireq: ireq.editable)
+        editable_ireq = next(iter(ireq for ireq in ireqs if ireq.editable), None)
         if editable_ireq:
-            yield editable_ireq  # ignore all the other specs: the editable one is the one that counts
+            yield editable_ireq  # only the editable match mattters, ignore all others
             continue
         ireqs = iter(ireqs)
         # deepcopy the accumulator so as to not modify the self.our_constraints invariant