Fix crashes and fails in forward references (#3952)

Forward references didn't work with anything apart from classes, for example 
this didn't work:

```
x: A
A = NamedTuple('A', [('x', int)])
```

The same situation was with `TypedDict`, `NewType`, and type aliases. The 
root problem is that these synthetic types are neither detected in first pass, 
nor fixed in third pass. In certain cases this can lead to crashes (first six issues 
below are various crash scenarios). This fixes these crashes by applying some 
additional patches after third pass.

Here is the summary of the PR:

* New simple wrapper type `ForwardRef` with only one field `link` is introduced 
  (with updates to type visitors)
* When an unknown type is found in second pass, the corresponding 
  `UnboundType` is wrapped in `ForwardRef`, it is given a "second chance" in 
  third pass.
* After third pass I record the "suspicious" nodes, where forward references and 
  synthetic types have been encountered and append patches (callbacks) to fix 
  them after third pass. Patches use the new visitor `TypeReplacer` (which is the 
  core of this PR).

Fixes #3340
Fixes #3419
Fixes #3674
Fixes #3685
Fixes #3799
Fixes #3836
Fixes #3881
Fixes #867
Fixes #2241
Fixes #2399
Fixes #1701
Fixes #3016
Fixes #3054
Fixes #2762
Fixes #3575
Fixes #3990

Author: ilevkivskyi

mypy/build.py

@@ -499,7 +499,7 @@ def __init__(self, data_dir: str,
         self.semantic_analyzer = SemanticAnalyzer(self.modules, self.missing_modules,
                                                   lib_path, self.errors, self.plugin)
         self.modules = self.semantic_analyzer.modules
-        self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors)
+        self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors, self.semantic_analyzer)
         self.all_types = {}  # type: Dict[Expression, Type]
         self.indirection_detector = TypeIndirectionVisitor()
         self.stale_modules = set()  # type: Set[str]
@@ -1730,10 +1730,13 @@ def semantic_analysis(self) -> None:
 
     def semantic_analysis_pass_three(self) -> None:
         assert self.tree is not None, "Internal error: method must be called on parsed file only"
+        patches = []  # type: List[Callable[[], None]]
         with self.wrap_context():
-            self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath, self.options)
+            self.manager.semantic_analyzer_pass3.visit_file(self.tree, self.xpath,
+                                                            self.options, patches)
             if self.options.dump_type_stats:
                 dump_type_stats(self.tree, self.xpath)
+        self.patches = patches + self.patches
 
     def semantic_analysis_apply_patches(self) -> None:
         for patch_func in self.patches:

mypy/indirection.py

@@ -101,3 +101,6 @@ def visit_ellipsis_type(self, t: types.EllipsisType) -> Set[str]:
 
     def visit_type_type(self, t: types.TypeType) -> Set[str]:
         return self._visit(t.item)
+
+    def visit_forwardref_type(self, t: types.ForwardRef) -> Set[str]:
+        return self._visit(t.link)

mypy/messages.py

@@ -20,7 +20,7 @@
 from mypy.types import (
     Type, CallableType, Instance, TypeVarType, TupleType, TypedDictType,
     UnionType, NoneTyp, AnyType, Overloaded, FunctionLike, DeletedType, TypeType,
-    UninhabitedType, TypeOfAny
+    UninhabitedType, TypeOfAny, ForwardRef, UnboundType
 )
 from mypy.nodes import (
     TypeInfo, Context, MypyFile, op_methods, FuncDef, reverse_type_aliases,
@@ -194,7 +194,7 @@ def quote_type_string(self, type_string: str) -> str:
         """Quotes a type representation for use in messages."""
         no_quote_regex = r'^<(tuple|union): \d+ items>$'
         if (type_string in ['Module', 'overloaded function', '<nothing>', '<deleted>']
-                or re.match(no_quote_regex, type_string) is not None):
+                or re.match(no_quote_regex, type_string) is not None or type_string.endswith('?')):
             # Messages are easier to read if these aren't quoted.  We use a
             # regex to match strings with variable contents.
             return type_string
@@ -309,6 +309,8 @@ def format_bare(self, typ: Type, verbosity: int = 0) -> str:
                 return '<nothing>'
         elif isinstance(typ, TypeType):
             return 'Type[{}]'.format(self.format_bare(typ.item, verbosity))
+        elif isinstance(typ, ForwardRef):  # may appear in semanal.py
+            return self.format_bare(typ.link, verbosity)
         elif isinstance(typ, FunctionLike):
             func = typ
             if func.is_type_obj():
@@ -350,6 +352,8 @@ def format_bare(self, typ: Type, verbosity: int = 0) -> str:
                 # function types may result in long and difficult-to-read
                 # error messages.
                 return 'overloaded function'
+        elif isinstance(typ, UnboundType):
+            return str(typ)
         elif typ is None:
             raise RuntimeError('Type is None')
         else:

mypy/nodes.py

@@ -1976,6 +1976,12 @@ class is generic then it will be a type constructor of higher kind.
     # Is this a newtype type?
     is_newtype = False
 
+    # If during analysis of ClassDef associated with this TypeInfo a syntethic
+    # type (NamedTuple or TypedDict) was generated, store the corresponding
+    # TypeInfo here. (This attribute does not need to be serialized, it is only
+    # needed during the semantic passes.)
+    replaced = None  # type: TypeInfo
+
     FLAGS = [
         'is_abstract', 'is_enum', 'fallback_to_any', 'is_named_tuple',
         'is_newtype', 'is_protocol', 'runtime_protocol'

mypy/semanal.py

@@ -11,7 +11,7 @@
 from mypy.types import (
     Type, Instance, AnyType, NoneTyp, TypeVisitor, CallableType, DeletedType, PartialType,
     TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType,
-    FunctionLike
+    FunctionLike, ForwardRef
 )
 from mypy.server.trigger import make_trigger
 
@@ -212,6 +212,9 @@ def visit_type_type(self, typ: TypeType) -> List[str]:
         # TODO: replace with actual implementation
         return []
 
+    def visit_forwardref_type(self, typ: ForwardRef) -> List[str]:
+        return get_type_dependencies(typ.link)
+
     def visit_type_var(self, typ: TypeVarType) -> List[str]:
         # TODO: replace with actual implementation
         return []