From 65c70fa5791b740b495fa92b3038306a87c18f4d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sat, 7 Mar 2020 22:35:37 -0500 Subject: [PATCH] Fix binary operator mapping in setup.py ast parser - Fix handling of `ast.binOp` nodes during unmapping of `setup.py` files when parsing -- add mapping for all binary operators and execute translated functions using the unparsed left and right hand sides of the binary operator - Add equivalent handling for `ast.Compare` types - Add `importlib.import_module` attempts to `ast.Compare` evaluations when encountering `ast.Attribute` types on either side of the comparison - Add handling for `ast.IfExp` type to evaluate truth values of the expression and return `node.body` if the expression is `True`, otherwise `node.orelse` - Add equivalence mapping for `ast.Ellipsis` type which becomes `ast.Constant` in python `>= 3.8` - Fixes #204 - Fixes #206 - Fixes #207 Signed-off-by: Dan Ryan --- .pre-commit-config.yaml | 15 ++- news/204.bugfix.rst | 1 + news/206.bugfix.rst | 1 + news/207.bugfix.rst | 1 + src/requirementslib/models/setup_info.py | 132 ++++++++++++++++++----- 5 files changed, 119 insertions(+), 31 deletions(-) create mode 100644 news/204.bugfix.rst create mode 100644 news/206.bugfix.rst create mode 100644 news/207.bugfix.rst diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f579267f..dde3109c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/ambv/black - rev: 19.10b0 + rev: stable hooks: - id: black args: ["--target-version=py27", "--target-version=py37"] @@ -11,11 +11,18 @@ repos: #- id: flake8 - repo: https://github.com/asottile/seed-isort-config - rev: v1.9.3 + rev: v2.1.0 hooks: - id: seed-isort-config + args: [ + --application-directories=src/requirementslib, + --settings-path=./, + # --exclude=tests/.*\.py, + # --exclude=src/requirementslib/models/setup_info.py + ] - - repo: https://github.com/pre-commit/mirrors-isort - rev: v4.3.21 + - repo: https://github.com/timothycrosley/isort + rev: 4.3.21 hooks: - id: isort + diff --git a/news/204.bugfix.rst b/news/204.bugfix.rst new file mode 100644 index 00000000..f20d30bf --- /dev/null +++ b/news/204.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue in binary operator mapping in the ``ast_parse_setup_py`` functionality of the dependency parser which could cause dependency resolution to fail. diff --git a/news/206.bugfix.rst b/news/206.bugfix.rst new file mode 100644 index 00000000..9795814d --- /dev/null +++ b/news/206.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue which caused mappings of binary operators to fail to evaluate when parsing ``setup.py`` files. diff --git a/news/207.bugfix.rst b/news/207.bugfix.rst new file mode 100644 index 00000000..8f2ca993 --- /dev/null +++ b/news/207.bugfix.rst @@ -0,0 +1 @@ +Fixed mapping and evaluation of boolean operators and comparisons when evaluating ``setup.py`` files with AST parser to discover dependencies. diff --git a/src/requirementslib/models/setup_info.py b/src/requirementslib/models/setup_info.py index da7078c6..8bf3496a 100644 --- a/src/requirementslib/models/setup_info.py +++ b/src/requirementslib/models/setup_info.py @@ -6,6 +6,7 @@ import contextlib import importlib import io +import operator import os import shutil import sys @@ -74,6 +75,7 @@ AnyStr, Sequence, ) + import requests from pip_shims.shims import InstallRequirement, PackageFinder from pkg_resources import ( PathMetadata, @@ -285,8 +287,11 @@ def get_extras_from_setupcfg(parser): return extras -def parse_setup_cfg(setup_cfg_contents, base_dir): - # type: (S, S) -> Dict[S, Union[S, None, Set[BaseRequirement], List[S], Dict[STRING_TYPE, Tuple[BaseRequirement]]]] +def parse_setup_cfg( + setup_cfg_contents, # type: S + base_dir, # type: S +): + # type: (...) -> Dict[S, Union[S, None, Set[BaseRequirement], List[S], Dict[STRING_TYPE, Tuple[BaseRequirement]]]] default_opts = { "metadata": {"name": "", "version": ""}, "options": { @@ -637,6 +642,37 @@ def get_metadata_from_dist(dist): } +AST_BINOP_MAP = dict(( + (ast.Add, operator.add), + (ast.Sub, operator.sub), + (ast.Mult, operator.mul), + (ast.Div, operator.floordiv), + (ast.Mod, operator.mod), + (ast.Pow, operator.pow), + (ast.LShift, operator.lshift), + (ast.RShift, operator.rshift), + (ast.BitAnd, operator.and_), + (ast.BitOr, operator.or_), + (ast.BitXor, operator.xor), +)) + + +AST_COMPARATORS = dict(( + (ast.Lt, operator.lt), + (ast.LtE, operator.le), + (ast.Eq, operator.eq), + (ast.Gt, operator.gt), + (ast.GtE, operator.ge), + (ast.NotEq, operator.ne), + (ast.Is, operator.is_), + (ast.IsNot, operator.is_not), + (ast.And, operator.and_), + (ast.Or, operator.or_), + (ast.Not, operator.not_), + (ast.In, operator.contains), +)) + + class Analyzer(ast.NodeVisitor): def __init__(self): self.name_types = [] @@ -661,10 +697,7 @@ def generic_visit(self, node): super(Analyzer, self).generic_visit(node) def visit_BinOp(self, node): - left = ast_unparse(node.left, initial_mapping=True) - right = ast_unparse(node.right, initial_mapping=True) - node.left = left - node.right = right + node = ast_unparse(node, initial_mapping=True) self.binOps.append(node) def unmap_binops(self): @@ -687,6 +720,10 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no unparse = partial( ast_unparse, initial_mapping=initial_mapping, analyzer=analyzer, recurse=recurse ) + if getattr(ast, "Constant", None): + constant = (ast.Constant, ast.Ellipsis) + else: + constant = ast.Ellipsis if isinstance(item, ast.Dict): unparsed = dict(zip(unparse(item.keys), unparse(item.values))) elif isinstance(item, ast.List): @@ -697,28 +734,26 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no unparsed = item.s elif isinstance(item, ast.Subscript): unparsed = unparse(item.value) + elif any(isinstance(item, k) for k in AST_BINOP_MAP.keys()): + unparsed = AST_BINOP_MAP[type(item)] elif isinstance(item, ast.BinOp): if analyzer and item in analyzer.binOps_map: unparsed = analyzer.binOps_map[item] - elif isinstance(item.op, ast.Add): + else: + right_item = unparse(item.right) + left_item = unparse(item.left) + op = getattr(item, "op", None) + op_func = unparse(op) if op is not None else op if not initial_mapping: - right_item = unparse(item.right) - left_item = unparse(item.left) - if not all( - isinstance(side, (six.string_types, int, float, list, tuple)) - for side in (left_item, right_item) - ): - item.left = left_item - item.right = right_item - unparsed = item - else: - unparsed = left_item + right_item + try: + unparsed = op_func(left_item, right_item) + except Exception: + unparsed = (left_item, op_func, right_item) else: + item.left = left_item + item.right = right_item + item.op = op_func unparsed = item - elif isinstance(item.op, ast.Sub): - unparsed = unparse(item.left) - unparse(item.right) - else: - unparsed = item elif isinstance(item, ast.Name): if not initial_mapping: unparsed = item.id @@ -735,6 +770,41 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no unparsed = item elif six.PY3 and isinstance(item, ast.NameConstant): unparsed = item.value + elif any(isinstance(item, k) for k in AST_COMPARATORS.keys()): + unparsed = AST_COMPARATORS[type(item)] + elif isinstance(item, constant): + unparsed = item.value + elif isinstance(item, ast.Compare): + if isinstance(item.left, ast.Attribute): + import importlib + + left = unparse(item.left) + if "." in left: + name, _, val = left.rpartition(".") + left = getattr(importlib.import_module(name), val, left) + comparators = [] + for comparator in item.comparators: + right = unparse(comparator) + if isinstance(comparator, ast.Attribute) and "." in right: + name, _, val = right.rpartition(".") + right = getattr(importlib.import_module(name), val, right) + comparators.append(right) + unparsed = (left, unparse(item.ops), comparators) + elif isinstance(item, ast.IfExp): + if initial_mapping: + unparsed = item + else: + left, ops, right = unparse(item.test) + truth_vals = [] + for i, op in enumerate(ops): + if i == 0: + truth_vals.append(op(left, right[i])) + else: + truth_vals.append(op(right[i - 1], right[i])) + if all(truth_vals): + unparsed = unparse(item.body) + else: + unparsed = unparse(item.orelse) elif isinstance(item, ast.Attribute): attr_name = getattr(item, "value", None) attr_attr = getattr(item, "attr", None) @@ -756,13 +826,21 @@ def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # no unparsed = name if not unparsed else unparsed elif isinstance(item, ast.Call): unparsed = {} - if isinstance(item.func, ast.Name): + if isinstance(item.func, (ast.Name, ast.Attribute)): func_name = unparse(item.func) - elif isinstance(item.func, ast.Attribute): - func_name = unparse(item.func) - if func_name: + else: + try: + func_name = unparse(item.func) + except Exception: + func_name = None + if isinstance(func_name, dict): + unparsed.update(func_name) + func_name = next(iter(func_name.keys())) + for keyword in getattr(item, "keywords", []): + unparsed[func_name].update(unparse(keyword)) + elif func_name: unparsed[func_name] = {} - for keyword in item.keywords: + for keyword in getattr(item, "keywords", []): unparsed[func_name].update(unparse(keyword)) elif isinstance(item, ast.keyword): unparsed = {unparse(item.arg): unparse(item.value)}