Basic support for ParamSpec type checking (#11594)

Add support for type checking several ParamSpec use cases (PEP 612).

@hauntsaninja previously added support for semantic analysis of ParamSpec
definitions, and this builds on top that foundation.

The implementation has these main things going on:
 * `ParamSpecType` that is similar to `TypeVarType` but has three "flavors" that
   correspond to `P`, `P.args` and `P.kwargs`
 * `CallableType` represents `Callable[P, T]` if the arguments are 
   (`*args: P.args`, `**kwargs: P.kwargs`) -- and more generally, there can also
   be arbitrary additional prefix arguments
 * Type variables of functions and classes can now be represented using
   `ParamSpecType` in addition to `TypeVarType`

There are still a bunch of TODOs. Some of these are important to address before the
release that includes this. I believe that this is good enough to merge and remaining 
issues can be fixed in follow-up PRs.

Notable missing features include these:
 * `Concatenate`
 * Specifying the value of ParamSpec explicitly (e.g. `Z[[int, str, bool]]`)
 * Various validity checks -- currently only some errors are caught
 * Special case of decorating a method (python/typeshed#6347)
 * `atexit.register(lambda: ...)` generates an error

Author: JukkaL

misc/proper_plugin.py

@@ -66,6 +66,7 @@ def is_special_target(right: ProperType) -> bool:
         if right.type_object().fullname in (
             'mypy.types.UnboundType',
             'mypy.types.TypeVarType',
+            'mypy.types.ParamSpecType',
             'mypy.types.RawExpressionType',
             'mypy.types.EllipsisType',
             'mypy.types.StarType',

mypy/applytype.py

@@ -5,7 +5,7 @@
 from mypy.expandtype import expand_type
 from mypy.types import (
     Type, TypeVarId, TypeVarType, CallableType, AnyType, PartialType, get_proper_types,
-    TypeVarLikeType, ProperType, ParamSpecType
+    TypeVarLikeType, ProperType, ParamSpecType, get_proper_type
 )
 from mypy.nodes import Context
 
@@ -18,9 +18,8 @@ def get_target_type(
     context: Context,
     skip_unsatisfied: bool
 ) -> Optional[Type]:
-    # TODO(PEP612): fix for ParamSpecType
     if isinstance(tvar, ParamSpecType):
-        return None
+        return type
     assert isinstance(tvar, TypeVarType)
     values = get_proper_types(tvar.values)
     if values:
@@ -90,6 +89,14 @@ def apply_generic_arguments(
         if target_type is not None:
             id_to_type[tvar.id] = target_type
 
+    param_spec = callable.param_spec()
+    if param_spec is not None:
+        nt = id_to_type.get(param_spec.id)
+        if nt is not None:
+            nt = get_proper_type(nt)
+            if isinstance(nt, CallableType):
+                callable = callable.expand_param_spec(nt)
+
     # Apply arguments to argument types.
     arg_types = [expand_type(at, id_to_type) for at in callable.arg_types]
 

mypy/checker.py

@@ -37,7 +37,8 @@
     UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType,
     is_named_instance, union_items, TypeQuery, LiteralType,
     is_optional, remove_optional, TypeTranslator, StarType, get_proper_type, ProperType,
-    get_proper_types, is_literal_type, TypeAliasType, TypeGuardedType)
+    get_proper_types, is_literal_type, TypeAliasType, TypeGuardedType, ParamSpecType
+)
 from mypy.sametypes import is_same_type
 from mypy.messages import (
     MessageBuilder, make_inferred_type_note, append_invariance_notes, pretty_seq,
@@ -976,13 +977,15 @@ def check_func_def(self, defn: FuncItem, typ: CallableType, name: Optional[str])
                                 ctx = typ
                             self.fail(message_registry.FUNCTION_PARAMETER_CANNOT_BE_COVARIANT, ctx)
                     if typ.arg_kinds[i] == nodes.ARG_STAR:
-                        # builtins.tuple[T] is typing.Tuple[T, ...]
-                        arg_type = self.named_generic_type('builtins.tuple',
-                                                           [arg_type])
+                        if not isinstance(arg_type, ParamSpecType):
+                            # builtins.tuple[T] is typing.Tuple[T, ...]
+                            arg_type = self.named_generic_type('builtins.tuple',
+                                                               [arg_type])
                     elif typ.arg_kinds[i] == nodes.ARG_STAR2:
-                        arg_type = self.named_generic_type('builtins.dict',
-                                                           [self.str_type(),
-                                                            arg_type])
+                        if not isinstance(arg_type, ParamSpecType):
+                            arg_type = self.named_generic_type('builtins.dict',
+                                                               [self.str_type(),
+                                                                arg_type])
                     item.arguments[i].variable.type = arg_type
 
                 # Type check initialization expressions.
@@ -1883,7 +1886,7 @@ def check_protocol_variance(self, defn: ClassDef) -> None:
                 expected = CONTRAVARIANT
             else:
                 expected = INVARIANT
-            if expected != tvar.variance:
+            if isinstance(tvar, TypeVarType) and expected != tvar.variance:
                 self.msg.bad_proto_variance(tvar.variance, tvar.name, expected, defn)
 
     def check_multiple_inheritance(self, typ: TypeInfo) -> None:

mypy/checkexpr.py

@@ -18,7 +18,7 @@
     Type, AnyType, CallableType, Overloaded, NoneType, TypeVarType,
     TupleType, TypedDictType, Instance, ErasedType, UnionType,
     PartialType, DeletedType, UninhabitedType, TypeType, TypeOfAny, LiteralType, LiteralValue,
-    is_named_instance, FunctionLike, ParamSpecType,
+    is_named_instance, FunctionLike, ParamSpecType, ParamSpecFlavor,
     StarType, is_optional, remove_optional, is_generic_instance, get_proper_type, ProperType,
     get_proper_types, flatten_nested_unions
 )
@@ -1025,11 +1025,31 @@ def check_callable_call(self,
             lambda i: self.accept(args[i]))
 
         if callee.is_generic():
+            need_refresh = any(isinstance(v, ParamSpecType) for v in callee.variables)
             callee = freshen_function_type_vars(callee)
             callee = self.infer_function_type_arguments_using_context(
                 callee, context)
             callee = self.infer_function_type_arguments(
                 callee, args, arg_kinds, formal_to_actual, context)
+            if need_refresh:
+                # Argument kinds etc. may have changed; recalculate actual-to-formal map
+                formal_to_actual = map_actuals_to_formals(
+                    arg_kinds, arg_names,
+                    callee.arg_kinds, callee.arg_names,
+                    lambda i: self.accept(args[i]))
+
+        param_spec = callee.param_spec()
+        if param_spec is not None and arg_kinds == [ARG_STAR, ARG_STAR2]:
+            arg1 = get_proper_type(self.accept(args[0]))
+            arg2 = get_proper_type(self.accept(args[1]))
+            if (is_named_instance(arg1, 'builtins.tuple')
+                    and is_named_instance(arg2, 'builtins.dict')):
+                assert isinstance(arg1, Instance)
+                assert isinstance(arg2, Instance)
+                if (isinstance(arg1.args[0], ParamSpecType)
+                        and isinstance(arg2.args[1], ParamSpecType)):
+                    # TODO: Check ParamSpec ids and flavors
+                    return callee.ret_type, callee
 
         arg_types = self.infer_arg_types_in_context(
             callee, args, arg_kinds, formal_to_actual)
@@ -3981,15 +4001,18 @@ def is_valid_var_arg(self, typ: Type) -> bool:
         return (isinstance(typ, TupleType) or
                 is_subtype(typ, self.chk.named_generic_type('typing.Iterable',
                                                             [AnyType(TypeOfAny.special_form)])) or
-                isinstance(typ, AnyType))
+                isinstance(typ, AnyType) or
+                (isinstance(typ, ParamSpecType) and typ.flavor == ParamSpecFlavor.ARGS))
 
     def is_valid_keyword_var_arg(self, typ: Type) -> bool:
         """Is a type valid as a **kwargs argument?"""
         ret = (
                 is_subtype(typ, self.chk.named_generic_type('typing.Mapping',
                     [self.named_type('builtins.str'), AnyType(TypeOfAny.special_form)])) or
                 is_subtype(typ, self.chk.named_generic_type('typing.Mapping',
-                    [UninhabitedType(), UninhabitedType()])))
+                    [UninhabitedType(), UninhabitedType()])) or
+                (isinstance(typ, ParamSpecType) and typ.flavor == ParamSpecFlavor.KWARGS)
+        )
         if self.chk.options.python_version[0] < 3:
             ret = ret or is_subtype(typ, self.chk.named_generic_type('typing.Mapping',
                 [self.named_type('builtins.unicode'), AnyType(TypeOfAny.special_form)]))

mypy/checkmember.py

@@ -6,7 +6,7 @@
 from mypy.types import (
     Type, Instance, AnyType, TupleType, TypedDictType, CallableType, FunctionLike,
     TypeVarLikeType, Overloaded, TypeVarType, UnionType, PartialType, TypeOfAny, LiteralType,
-    DeletedType, NoneType, TypeType, has_type_vars, get_proper_type, ProperType
+    DeletedType, NoneType, TypeType, has_type_vars, get_proper_type, ProperType, ParamSpecType
 )
 from mypy.nodes import (
     TypeInfo, FuncBase, Var, FuncDef, SymbolNode, SymbolTable, Context,
@@ -669,6 +669,9 @@ def f(self: S) -> T: ...
             selfarg = item.arg_types[0]
             if subtypes.is_subtype(dispatched_arg_type, erase_typevars(erase_to_bound(selfarg))):
                 new_items.append(item)
+            elif isinstance(selfarg, ParamSpecType):
+                # TODO: This is not always right. What's the most reasonable thing to do here?
+                new_items.append(item)
     if not new_items:
         # Choose first item for the message (it may be not very helpful for overloads).
         msg.incompatible_self_argument(name, dispatched_arg_type, items[0],

mypy/constraints.py

@@ -7,7 +7,8 @@
     CallableType, Type, TypeVisitor, UnboundType, AnyType, NoneType, TypeVarType, Instance,
     TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType, DeletedType,
     UninhabitedType, TypeType, TypeVarId, TypeQuery, is_named_instance, TypeOfAny, LiteralType,
-    ProperType, get_proper_type, TypeAliasType, is_union_with_any
+    ProperType, ParamSpecType, get_proper_type, TypeAliasType, is_union_with_any,
+    callable_with_ellipsis
 )
 from mypy.maptype import map_instance_to_supertype
 import mypy.subtypes
@@ -398,6 +399,10 @@ def visit_type_var(self, template: TypeVarType) -> List[Constraint]:
         assert False, ("Unexpected TypeVarType in ConstraintBuilderVisitor"
                        " (should have been handled in infer_constraints)")
 
+    def visit_param_spec(self, template: ParamSpecType) -> List[Constraint]:
+        # Can't infer ParamSpecs from component values (only via Callable[P, T]).
+        return []
+
     # Non-leaf types
 
     def visit_instance(self, template: Instance) -> List[Constraint]:
@@ -438,14 +443,16 @@ def visit_instance(self, template: Instance) -> List[Constraint]:
                 # N.B: We use zip instead of indexing because the lengths might have
                 # mismatches during daemon reprocessing.
                 for tvar, mapped_arg, instance_arg in zip(tvars, mapped.args, instance.args):
-                    # The constraints for generic type parameters depend on variance.
-                    # Include constraints from both directions if invariant.
-                    if tvar.variance != CONTRAVARIANT:
-                        res.extend(infer_constraints(
-                            mapped_arg, instance_arg, self.direction))
-                    if tvar.variance != COVARIANT:
-                        res.extend(infer_constraints(
-                            mapped_arg, instance_arg, neg_op(self.direction)))
+                    # TODO: ParamSpecType
+                    if isinstance(tvar, TypeVarType):
+                        # The constraints for generic type parameters depend on variance.
+                        # Include constraints from both directions if invariant.
+                        if tvar.variance != CONTRAVARIANT:
+                            res.extend(infer_constraints(
+                                mapped_arg, instance_arg, self.direction))
+                        if tvar.variance != COVARIANT:
+                            res.extend(infer_constraints(
+                                mapped_arg, instance_arg, neg_op(self.direction)))
                 return res
             elif (self.direction == SUPERTYPE_OF and
                     instance.type.has_base(template.type.fullname)):
@@ -454,14 +461,16 @@ def visit_instance(self, template: Instance) -> List[Constraint]:
                 # N.B: We use zip instead of indexing because the lengths might have
                 # mismatches during daemon reprocessing.
                 for tvar, mapped_arg, template_arg in zip(tvars, mapped.args, template.args):
-                    # The constraints for generic type parameters depend on variance.
-                    # Include constraints from both directions if invariant.
-                    if tvar.variance != CONTRAVARIANT:
-                        res.extend(infer_constraints(
-                            template_arg, mapped_arg, self.direction))
-                    if tvar.variance != COVARIANT:
-                        res.extend(infer_constraints(
-                            template_arg, mapped_arg, neg_op(self.direction)))
+                    # TODO: ParamSpecType
+                    if isinstance(tvar, TypeVarType):
+                        # The constraints for generic type parameters depend on variance.
+                        # Include constraints from both directions if invariant.
+                        if tvar.variance != CONTRAVARIANT:
+                            res.extend(infer_constraints(
+                                template_arg, mapped_arg, self.direction))
+                        if tvar.variance != COVARIANT:
+                            res.extend(infer_constraints(
+                                template_arg, mapped_arg, neg_op(self.direction)))
                 return res
             if (template.type.is_protocol and self.direction == SUPERTYPE_OF and
                     # We avoid infinite recursion for structural subtypes by checking
@@ -536,32 +545,48 @@ def infer_constraints_from_protocol_members(self,
 
     def visit_callable_type(self, template: CallableType) -> List[Constraint]:
         if isinstance(self.actual, CallableType):
-            cactual = self.actual
-            # FIX verify argument counts
-            # FIX what if one of the functions is generic
             res: List[Constraint] = []
+            cactual = self.actual
+            param_spec = template.param_spec()
+            if param_spec is None:
+                # FIX verify argument counts
+                # FIX what if one of the functions is generic
+
+                # We can't infer constraints from arguments if the template is Callable[..., T]
+                # (with literal '...').
+                if not template.is_ellipsis_args:
+                    # The lengths should match, but don't crash (it will error elsewhere).
+                    for t, a in zip(template.arg_types, cactual.arg_types):
+                        # Negate direction due to function argument type contravariance.
+                        res.extend(infer_constraints(t, a, neg_op(self.direction)))
+            else:
+                # TODO: Direction
+                # TODO: Deal with arguments that come before param spec ones?
+                res.append(Constraint(param_spec.id,
+                                      SUBTYPE_OF,
+                                      cactual.copy_modified(ret_type=NoneType())))
 
-            # We can't infer constraints from arguments if the template is Callable[..., T] (with
-            # literal '...').
-            if not template.is_ellipsis_args:
-                # The lengths should match, but don't crash (it will error elsewhere).
-                for t, a in zip(template.arg_types, cactual.arg_types):
-                    # Negate direction due to function argument type contravariance.
-                    res.extend(infer_constraints(t, a, neg_op(self.direction)))
             template_ret_type, cactual_ret_type = template.ret_type, cactual.ret_type
             if template.type_guard is not None:
                 template_ret_type = template.type_guard
             if cactual.type_guard is not None:
                 cactual_ret_type = cactual.type_guard
+
             res.extend(infer_constraints(template_ret_type, cactual_ret_type,
                                          self.direction))
             return res
         elif isinstance(self.actual, AnyType):
-            # FIX what if generic
-            res = self.infer_against_any(template.arg_types, self.actual)
+            param_spec = template.param_spec()
             any_type = AnyType(TypeOfAny.from_another_any, source_any=self.actual)
-            res.extend(infer_constraints(template.ret_type, any_type, self.direction))
-            return res
+            if param_spec is None:
+                # FIX what if generic
+                res = self.infer_against_any(template.arg_types, self.actual)
+                res.extend(infer_constraints(template.ret_type, any_type, self.direction))
+                return res
+            else:
+                return [Constraint(param_spec.id,
+                                   SUBTYPE_OF,
+                                   callable_with_ellipsis(any_type, any_type, template.fallback))]
         elif isinstance(self.actual, Overloaded):
             return self.infer_against_overloaded(self.actual, template)
         elif isinstance(self.actual, TypeType):

mypy/erasetype.py

@@ -4,7 +4,7 @@
     Type, TypeVisitor, UnboundType, AnyType, NoneType, TypeVarId, Instance, TypeVarType,
     CallableType, TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType,
     DeletedType, TypeTranslator, UninhabitedType, TypeType, TypeOfAny, LiteralType, ProperType,
-    get_proper_type, TypeAliasType
+    get_proper_type, TypeAliasType, ParamSpecType
 )
 from mypy.nodes import ARG_STAR, ARG_STAR2
 
@@ -57,6 +57,9 @@ def visit_instance(self, t: Instance) -> ProperType:
     def visit_type_var(self, t: TypeVarType) -> ProperType:
         return AnyType(TypeOfAny.special_form)
 
+    def visit_param_spec(self, t: ParamSpecType) -> ProperType:
+        return AnyType(TypeOfAny.special_form)
+
     def visit_callable_type(self, t: CallableType) -> ProperType:
         # We must preserve the fallback type for overload resolution to work.
         any_type = AnyType(TypeOfAny.special_form)
@@ -125,6 +128,11 @@ def visit_type_var(self, t: TypeVarType) -> Type:
             return self.replacement
         return t
 
+    def visit_param_spec(self, t: ParamSpecType) -> Type:
+        if self.erase_id(t.id):
+            return self.replacement
+        return t
+
     def visit_type_alias_type(self, t: TypeAliasType) -> Type:
         # Type alias target can't contain bound type variables, so
         # it is safe to just erase the arguments.