Add 'If' condition evaluation feature based on user input

docs/source/codedoc.rst

@@ -318,3 +318,42 @@ The content of ``my_project/__init__.py`` includes::
     from .core._impl import MyClass
 
     __all__ = ("MyClass",)
+
+.. _branch-priorities:
+
+Branch priorities
+-----------------
+
+When pydoctor deals with try/except/else or if/else block, it makes sure that the names defined in 
+the "principal" branch do not get overriden by names defined in the except hanlders or ifs' else block. 
+
+Meaning that in the context of the code below, ``ssl`` would resolve to ``twisted.internet.ssl``:
+
+.. code:: python
+
+    try:
+        from twisted.internet import ssl as _ssl
+    except ImportError:
+        ssl = None
+    else:
+        ssl = _ssl
+
+Similarly, in the context of the code below, the first ``CapSys`` class will be 
+documented and the second one will be ignored.
+
+.. code:: python
+
+    from typing import TYPE_CHECKING
+    if TYPE_CHECKING:
+        from typing import Protocol
+        class CapSys(Protocol):
+            def readouterr() -> Any:
+                ...
+    else:
+        class CapSys(object): # ignored
+            ...
+
+But sometimes pydoctor can be better off analysing the ``TYPE_CHECKING`` blocks and should 
+stick to the runtime version of the code instead. 
+See the :ref:`Tweak ifs branch priorities <tweak-ifs-branch-priority>` section for more 
+informatons about how to handle this kind of corner cases.

docs/source/customize.rst

@@ -105,6 +105,8 @@ Privacy tweak examples
 .. note:: Quotation marks should be added around each rule to avoid shell expansions.
     Unless the arguments are passed directly to pydoctor, like in Sphinx's ``conf.py``, in this case you must not quote the privacy rules.
 
+.. _use-custom-system-class:
+
 Use a custom system class
 -------------------------
 
@@ -116,7 +118,8 @@ and pass your custom class dotted name with the following argument::
 System class allows you to customize certain aspect of the system and configure the enabled extensions. 
 If what you want to achieve has something to do with the state of some objects in the Documentable tree, 
 it's very likely that you can do it without the need to override any system method, 
-by using the extension mechanism described below.
+by using the extension mechanism described below. Configuring extenions and other forms of system customizations
+does, nonetheless requires you to subclass the :py:class:`pydoctor.model.System` and override the required class variables.
 
 Brief on pydoctor extensions
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -218,6 +221,65 @@ if some want to write a custom :py:class:`pydoctor.sphinx.SphinxInventory`.
     If you feel like other users of the community might benefit from your extension as well, please 
     don't hesitate to open a pull request adding your extension module to the package :py:mod:`pydoctor.extensions`.
 
+.. _tweak-ifs-branch-priority:
+
+Tweak 'ifs' branch priority
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Sometimes pydoctor default :ref:`Branch priorities <branch-priorities>` can be inconvenient. 
+Specifically, when dealing with ``TYPE_CHECKING``. 
+A common purpose of ``if TYPE_CHECKING:`` blocks is to contain imports that are only 
+necessary for the purpose of type annotations. These might be circular imports at runtime. 
+When these types are used to annotate arguments or return values, 
+they are relevant to pydoctor as well: either to extract type information 
+from annotations or to look up the fully qualified version of some ``L{name}``.
+
+In other cases, ``if TYPE_CHECKING:`` blocks are used to perform trickery 
+to make ``mypy`` accept code that is difficult to analyze. 
+In these cases, pydoctor can have better results by analysing the runtime version of the code instead. 
+For details, read `the comments in this Klein PR <https://github.com/twisted/klein/pull/315>`_.
+
+Pydoctor has the ability to evaluate some simple name based :py:class:`ast.If` 
+condition in order to visit either the statements in main ``If.body`` or the ``If.orelse`` only.
+
+You can instrut pydoctor do to such things with a custom system class, 
+by overriding the class variable :py:attr:`pydoctor.model.System.eval_if`.
+This variable should be a dict of strings to dict of strings to any value. 
+This mechanism currently supports simple name based 'Ifs' only.
+
+Meaning like::
+
+    if TYPE_CHECKING:
+        ...
+    if typing.TYPE_CHECKING:
+        ...
+
+Does not recognize 'If' expressions like::
+
+    if TYPE_CHECKING==True:
+        ...
+    if TYPE_CHECKING is not False:
+        ...
+
+The code below demonstrate an example usage. 
+
+The assignment to :py:attr:`pydoctor.model.System.eval_if` declares two custom 'If' evaluation rules. 
+The first rule applies only to the 'Ifs' in ``my_mod`` and second one applies to all objects directly in the package ``my_mod._complex``
+Both rules makes the statements in ``if TYPE_CHECKING:`` blocks skipped to give the priority to what's defined in the ``else:`` blocks.
+
+.. code:: python
+
+    class MySystem(model.System):
+        eval_if = {
+            'my_mod':{'TYPE_CHECKING':False},
+            'my_mod._complex.*':{'TYPE_CHECKING':False},
+        }
+
+.. note:: See :py:mod:`pydoctor.qnmatch` for more informations regarding the pattern syntax. 
+
+Then use the ``--system-class`` option as described in :ref:`Use a custom system class <use-custom-system-class>` section.
+
+
 Use a custom writer class
 -------------------------
 

pydoctor/astbuilder.py

@@ -1,6 +1,7 @@
 """Convert ASTs into L{pydoctor.model.Documentable} instances."""
 
 import ast
+import contextlib
 import sys
 from attr import attrs, attrib
 from functools import partial
@@ -13,9 +14,9 @@
 )
 
 import astor
-from pydoctor import epydoc2stan, model, node2stan
+from pydoctor import epydoc2stan, model, node2stan, qnmatch
 from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval
-from pydoctor.astutils import bind_args, node2dottedname, node2fullname, is__name__equals__main__, NodeVisitor
+from pydoctor.astutils import evaluate_If_test, bind_args, node2dottedname, node2fullname, is__name__equals__main__, NodeVisitor
 
 def parseFile(path: Path) -> ast.Module:
     """Parse the contents of a Python source file."""
@@ -194,22 +195,118 @@ def extract_final_subscript(annotation: ast.Subscript) -> ast.expr:
         assert isinstance(ann_slice, ast.expr)
         return ann_slice
 
+def getframe(ob:'model.Documentable') -> model.CanContainImportsDocumentable:
+    """
+    Returns the first L{model.Class} or L{model.Module} starting 
+    at C{ob} and going upward in the tree.
+    """
+    if isinstance(ob, model.Inheritable):
+        return getframe(ob.parent)
+    else:
+        assert isinstance(ob, model.CanContainImportsDocumentable)
+        return ob
+
+def get_eval_if(ob: 'model.Documentable') -> Dict[str, Any]:
+    """
+    Returns the If evaluation rules applicable in the context of the received object.
+    """
+    global_eval_if = ob.system.eval_if
+    ob_eval_if: Dict[str, Any] = {}
+    for p in global_eval_if.keys():
+        if qnmatch.qnmatch(ob.fullName(), p):
+            ob_eval_if.update(global_eval_if[p])
+    return ob_eval_if
+
+
 class ModuleVistor(NodeVisitor):
 
     def __init__(self, builder: 'ASTBuilder', module: model.Module):
         super().__init__()
         self.builder = builder
         self.system = builder.system
         self.module = module
+        self._override_guard_state: Tuple[bool, model.Documentable, Sequence[str]] = \
+                                    (False, cast(model.Documentable, None), [])
+
+    @contextlib.contextmanager
+    def override_guard(self) -> Iterator[None]:
+        """
+        Returns a context manager that will make the builder ignore any extraneous 
+        assigments to existing names within the same context.  Currently used to visit C{If.orelse} and C{Try.handlers}.
+
+        @note: The list of existing names is generated at the moment of
+            calling the function, such that new names defined inside these blocks follows the usual override rules.
+        """
+        ctx = getframe(self.builder.current)
+        ignore_override_init = self._override_guard_state
+        # we list names only once to ignore new names added inside the block,
+        # they should be overriden as usual.
+        self._override_guard_state = (True, ctx, self._list_names(ctx))
+        yield
+        self._override_guard_state = ignore_override_init
+
+    def _list_names(self, ob: model.Documentable) -> List[str]:
+        """
+        List the names currently defined in a class/module.
+        """
+        names = list(ob.contents)
+        if isinstance(ob, model.CanContainImportsDocumentable):
+            names.extend(ob._localNameToFullName_map)
+        return names
 
+    def _name_in_override_guard(self, ob: model.Documentable, name:str) -> bool:
+        """Should this name be ignored because it matches the override guard in the context of C{ob}?"""
+        return self._override_guard_state[0] is True \
+            and self._override_guard_state[1] is ob \
+                and name in self._override_guard_state[2]
 
     def visit_If(self, node: ast.If) -> None:
         if isinstance(node.test, ast.Compare):
             if is__name__equals__main__(node.test):
                 # skip if __name__ == '__main__': blocks since
                 # whatever is declared in them cannot be imported
                 # and thus is not part of the API
-                raise self.SkipNode()
+                raise self.SkipChildren() # we use SkipChildren() here to still call depart().
+
+        # 'ast.If' evaluation feature, do not visit one of the branches based
+        # on user customizations, else visit both branches as ususal.
+        if not self.system.eval_if:
+            return
+        current = self.builder.current
+        if_eval_in_this_context = get_eval_if(current)
+        if not if_eval_in_this_context:
+            return
+
+        evaluated = evaluate_If_test(node, if_eval_in_this_context, current)
+
+        if evaluated == False:
+            # this will only call depart(), and thus visit only on the else branch of the If.
+            raise self.SkipChildren()
+        if evaluated == True:
+            # this will skip the call to depart(), so the else branch will not be visited at all.
+            raise self.SkipDeparture()
+
+    def depart_If(self, node: ast.If) -> None:
+        # At this point the body of the Try node has already been visited
+        # Visit the 'orelse' block of the If node, with override guard
+        with self.override_guard():
+            for n in node.orelse:
+                self.walkabout(n)
+
+    def depart_Try(self, node: ast.Try) -> None:
+        # At this point the body of the Try node has already been visited
+        # Visit the 'orelse' and 'finalbody' blocks of the Try node.
+
+        for n in node.orelse:
+            self.walkabout(n)
+        for n in node.finalbody:
+            self.walkabout(n)
+
+        # Visit the handlers with override guard 
+        with self.override_guard():
+            for h in node.handlers:
+                for n in h.body:
+                    self.walkabout(n)
 
     def visit_Module(self, node: ast.Module) -> None:
         assert self.module.docstring is None
@@ -227,6 +324,9 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None:
         parent = self.builder.current
         if isinstance(parent, model.Function):
             raise self.SkipNode()
+        # Ignore in override guard
+        if self._name_in_override_guard(parent, node.name):
+            raise self.SkipNode()
 
         rawbases = []
         bases = []
@@ -328,6 +428,11 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
     def _importAll(self, modname: str) -> None:
         """Handle a C{from <modname> import *} statement."""
 
+        # Always ignore import * in override guard
+        if self._override_guard_state[0]:
+            self.builder.warning("ignored import *", modname)
+            return
+
         mod = self.system.getProcessedModule(modname)
         if mod is None:
             # We don't have any information about the module, so we don't know
@@ -420,7 +525,11 @@ def _importNames(self, modname: str, names: Iterable[ast.alias]) -> None:
             orgname, asname = al.name, al.asname
             if asname is None:
                 asname = orgname
-
+
+            # Ignore in override guard
+            if self._name_in_override_guard(current, asname):
+                continue
+
             if mod is not None and self._handleReExport(exports, orgname, asname, mod) is True:
                 continue
 
@@ -449,9 +558,14 @@ def visit_Import(self, node: ast.Import) -> None:
                                  str(self.builder.current))
             return
         _localNameToFullName = self.builder.current._localNameToFullName_map
+        current = self.builder.current
         for al in node.names:
             fullname, asname = al.name, al.asname
+            # Ignore in override guard
+            if self._name_in_override_guard(current, asname or fullname):
+                continue
             if asname is not None:
+                # Do not create an alias with the same name as value
                 _localNameToFullName[asname] = fullname
 
 
@@ -638,6 +752,8 @@ def _handleInstanceVar(self,
             return
         if not _maybeAttribute(cls, name):
             return
+        if self._name_in_override_guard(cls, name):
+            return
 
         # Class variables can only be Attribute, so it's OK to cast because we used _maybeAttribute() above.
         obj = cast(Optional[model.Attribute], cls.contents.get(name))
@@ -729,6 +845,8 @@ def _handleAssignment(self,
         if isinstance(targetNode, ast.Name):
             target = targetNode.id
             scope = self.builder.current
+            if self._name_in_override_guard(scope, target):
+                return
             if isinstance(scope, model.Module):
                 self._handleAssignmentInModule(target, annotation, expr, lineno)
             elif isinstance(scope, model.Class):
@@ -788,6 +906,9 @@ def _handleFunctionDef(self,
         parent = self.builder.current
         if isinstance(parent, model.Function):
             raise self.SkipNode()
+        # Ignore in override guard
+        if self._name_in_override_guard(parent, node.name):
+            raise self.SkipNode()
 
         lineno = node.lineno
 

pydoctor/astutils.py

@@ -4,7 +4,7 @@
 
 import sys
 from numbers import Number
-from typing import Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING
+from typing import Any, Dict, Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING
 from inspect import BoundArguments, Signature
 import ast
 
@@ -155,3 +155,30 @@ def is_using_annotations(expr: Optional[ast.AST],
             if full_name in annotations:
                 return True
     return False
+
+def evaluate_If_test(node:ast.If, eval_if:Dict[str, Any], ctx:'model.Documentable') -> Optional[bool]:
+    """
+    Currently supports Ifs with dotted names only.
+
+    Like::
+
+        if TYPE_CHECKING:
+            ...
+        if typing.TYPE_CHECKING:
+            ...
+
+    Does not recognize stuff like::
+
+        if TYPE_CHECKING==True:
+            ...
+        if TYPE_CHECKING is not False:
+            ...
+    """
+    _test = node.test
+    # Supports simple dotted names only for now.
+    # But if we were to add more support for comparators,
+    # this is where it should happend.
+    full_name = node2fullname(_test, ctx)
+    if full_name and full_name in eval_if:
+        return bool(eval_if[full_name])
+    return None