Fix binary operator mapping in setup.py ast parser

- Fix handling of `ast.binOp` nodes during unmapping of `setup.py`
  files when parsing -- add mapping for all binary operators and
  execute translated functions using the unparsed left and right hand
  sides of the binary operator
- Add equivalent handling for `ast.Compare` types
- Add `importlib.import_module` attempts to `ast.Compare` evaluations
  when encountering `ast.Attribute` types on either side of the
  comparison
- Add handling for `ast.IfExp` type to evaluate truth values of the
  expression and return `node.body` if the expression is `True`,
  otherwise `node.orelse`
- Add equivalence mapping for `ast.Ellipsis` type which becomes
  `ast.Constant` in python `>= 3.8`
- Fixes #204
- Fixes #206
- Fixes #207

Signed-off-by: Dan Ryan <[email protected]>

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/ambv/black
-    rev: 19.10b0
+    rev: stable
     hooks:
       - id: black
         args: ["--target-version=py27", "--target-version=py37"]
@@ -11,11 +11,18 @@ repos:
       #- id: flake8
 
   - repo: https://github.com/asottile/seed-isort-config
-    rev: v1.9.3
+    rev: v2.1.0
     hooks:
       - id: seed-isort-config
+        args: [
+          --application-directories=src/requirementslib,
+          --settings-path=./,
+          # --exclude=tests/.*\.py,
+          # --exclude=src/requirementslib/models/setup_info.py
+        ]
 
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v4.3.21
+  - repo: https://github.com/timothycrosley/isort
+    rev: 4.3.21
     hooks:
       - id: isort
+

news/204.bugfix.rst

@@ -0,0 +1 @@
+Fixed an issue in binary operator mapping in the ``ast_parse_setup_py`` functionality of the dependency parser which could cause dependency resolution to fail.

news/206.bugfix.rst

@@ -0,0 +1 @@
+Fixed an issue which caused mappings of binary operators to fail to evaluate when parsing ``setup.py`` files.

news/207.bugfix.rst

@@ -0,0 +1 @@
+Fixed mapping and evaluation of boolean operators and comparisons when evaluating ``setup.py`` files with AST parser to discover dependencies.

src/requirementslib/models/setup_info.py

@@ -6,6 +6,7 @@
 import contextlib
 import importlib
 import io
+import operator
 import os
 import shutil
 import sys
@@ -74,6 +75,7 @@
         AnyStr,
         Sequence,
     )
+    import requests
     from pip_shims.shims import InstallRequirement, PackageFinder
     from pkg_resources import (
         PathMetadata,
@@ -285,8 +287,11 @@ def get_extras_from_setupcfg(parser):
     return extras
 
 
-def parse_setup_cfg(setup_cfg_contents, base_dir):
-    # type: (S, S) -> Dict[S, Union[S, None, Set[BaseRequirement], List[S], Dict[STRING_TYPE, Tuple[BaseRequirement]]]]
+def parse_setup_cfg(
+    setup_cfg_contents,  # type: S
+    base_dir,  # type: S
+):
+    # type: (...) -> Dict[S, Union[S, None, Set[BaseRequirement], List[S], Dict[STRING_TYPE, Tuple[BaseRequirement]]]]
     default_opts = {
         "metadata": {"name": "", "version": ""},
         "options": {
@@ -637,6 +642,37 @@ def get_metadata_from_dist(dist):
     }
 
 
+AST_BINOP_MAP = dict((
+    (ast.Add, operator.add),
+    (ast.Sub, operator.sub),
+    (ast.Mult, operator.mul),
+    (ast.Div, operator.floordiv),
+    (ast.Mod, operator.mod),
+    (ast.Pow, operator.pow),
+    (ast.LShift, operator.lshift),
+    (ast.RShift, operator.rshift),
+    (ast.BitAnd, operator.and_),
+    (ast.BitOr, operator.or_),
+    (ast.BitXor, operator.xor),
+))
+
+
+AST_COMPARATORS = dict((
+    (ast.Lt, operator.lt),
+    (ast.LtE, operator.le),
+    (ast.Eq, operator.eq),
+    (ast.Gt, operator.gt),
+    (ast.GtE, operator.ge),
+    (ast.NotEq, operator.ne),
+    (ast.Is, operator.is_),
+    (ast.IsNot, operator.is_not),
+    (ast.And, operator.and_),
+    (ast.Or, operator.or_),
+    (ast.Not, operator.not_),
+    (ast.In, operator.contains),
+))
+
+
 class Analyzer(ast.NodeVisitor):
     def __init__(self):
         self.name_types = []
@@ -661,10 +697,7 @@ def generic_visit(self, node):
         super(Analyzer, self).generic_visit(node)
 
     def visit_BinOp(self, node):
-        left = ast_unparse(node.left, initial_mapping=True)
-        right = ast_unparse(node.right, initial_mapping=True)
-        node.left = left
-        node.right = right
+        node = ast_unparse(node, initial_mapping=True)
         self.binOps.append(node)
 
     def unmap_binops(self):
@@ -687,6 +720,10 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True):  # no
     unparse = partial(
         ast_unparse, initial_mapping=initial_mapping, analyzer=analyzer, recurse=recurse
     )
+    if getattr(ast, "Constant", None):
+        constant = (ast.Constant, ast.Ellipsis)
+    else:
+        constant = ast.Ellipsis
     if isinstance(item, ast.Dict):
         unparsed = dict(zip(unparse(item.keys), unparse(item.values)))
     elif isinstance(item, ast.List):
@@ -697,28 +734,26 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True):  # no
         unparsed = item.s
     elif isinstance(item, ast.Subscript):
         unparsed = unparse(item.value)
+    elif any(isinstance(item, k) for k in AST_BINOP_MAP.keys()):
+        unparsed = AST_BINOP_MAP[type(item)]
     elif isinstance(item, ast.BinOp):
         if analyzer and item in analyzer.binOps_map:
             unparsed = analyzer.binOps_map[item]
-        elif isinstance(item.op, ast.Add):
+        else:
+            right_item = unparse(item.right)
+            left_item = unparse(item.left)
+            op = getattr(item, "op", None)
+            op_func = unparse(op) if op is not None else op
             if not initial_mapping:
-                right_item = unparse(item.right)
-                left_item = unparse(item.left)
-                if not all(
-                    isinstance(side, (six.string_types, int, float, list, tuple))
-                    for side in (left_item, right_item)
-                ):
-                    item.left = left_item
-                    item.right = right_item
-                    unparsed = item
-                else:
-                    unparsed = left_item + right_item
+                try:
+                    unparsed = op_func(left_item, right_item)
+                except Exception:
+                    unparsed = (left_item, op_func, right_item)
             else:
+                item.left = left_item
+                item.right = right_item
+                item.op = op_func
                 unparsed = item
-        elif isinstance(item.op, ast.Sub):
-            unparsed = unparse(item.left) - unparse(item.right)
-        else:
-            unparsed = item
     elif isinstance(item, ast.Name):
         if not initial_mapping:
             unparsed = item.id
@@ -735,6 +770,41 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True):  # no
             unparsed = item
     elif six.PY3 and isinstance(item, ast.NameConstant):
         unparsed = item.value
+    elif any(isinstance(item, k) for k in AST_COMPARATORS.keys()):
+        unparsed = AST_COMPARATORS[type(item)]
+    elif isinstance(item, constant):
+        unparsed = item.value
+    elif isinstance(item, ast.Compare):
+        if isinstance(item.left, ast.Attribute):
+            import importlib
+
+            left = unparse(item.left)
+            if "." in left:
+                name, _, val = left.rpartition(".")
+                left = getattr(importlib.import_module(name), val, left)
+            comparators = []
+            for comparator in item.comparators:
+                right = unparse(comparator)
+                if isinstance(comparator, ast.Attribute) and "." in right:
+                    name, _, val = right.rpartition(".")
+                    right = getattr(importlib.import_module(name), val, right)
+                comparators.append(right)
+        unparsed = (left, unparse(item.ops), comparators)
+    elif isinstance(item, ast.IfExp):
+        if initial_mapping:
+            unparsed = item
+        else:
+            left, ops, right = unparse(item.test)
+            truth_vals = []
+            for i, op in enumerate(ops):
+                if i == 0:
+                    truth_vals.append(op(left, right[i]))
+                else:
+                    truth_vals.append(op(right[i - 1], right[i]))
+            if all(truth_vals):
+                unparsed = unparse(item.body)
+            else:
+                unparsed = unparse(item.orelse)
     elif isinstance(item, ast.Attribute):
         attr_name = getattr(item, "value", None)
         attr_attr = getattr(item, "attr", None)
@@ -756,13 +826,21 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True):  # no
             unparsed = name if not unparsed else unparsed
     elif isinstance(item, ast.Call):
         unparsed = {}
-        if isinstance(item.func, ast.Name):
+        if isinstance(item.func, (ast.Name, ast.Attribute)):
             func_name = unparse(item.func)
-        elif isinstance(item.func, ast.Attribute):
-            func_name = unparse(item.func)
-        if func_name:
+        else:
+            try:
+                func_name = unparse(item.func)
+            except Exception:
+                func_name = None
+        if isinstance(func_name, dict):
+            unparsed.update(func_name)
+            func_name = next(iter(func_name.keys()))
+            for keyword in getattr(item, "keywords", []):
+                unparsed[func_name].update(unparse(keyword))
+        elif func_name:
             unparsed[func_name] = {}
-            for keyword in item.keywords:
+            for keyword in getattr(item, "keywords", []):
                 unparsed[func_name].update(unparse(keyword))
     elif isinstance(item, ast.keyword):
         unparsed = {unparse(item.arg): unparse(item.value)}