Added tests for the "type qualifiers" section of the typing spec

README.md

@@ -26,6 +26,8 @@ This GitHub repository is used for several things:
 - A [discussion forum](https://github.com/python/typing/discussions) for typing-related user
   help is hosted here.
 
+- [Conformance test](https://github.com/python/typing/blob/main/conformance/README.md) for Python static type checkers. The [latest conformance test results](https://htmlpreview.github.io/?https://github.com/python/typing/blob/main/conformance/results/results.html) are here.
+
 Historically, this repository also hosted:
 
 - The `typing_extensions` package, which now lives in the

conformance/results/mypy/annotations_coroutines.toml

@@ -0,0 +1,3 @@
+conformant = "Pass"
+output = """
+"""

conformance/results/mypy/annotations_forward_refs.toml

@@ -0,0 +1,31 @@
+conformant = "Partial"
+notes = """
+Does not report error for a forward reference that is not enclosed in quotes.
+Does not report error for use of quoted type with "|" operator (runtime error).
+Incorrectly generates error for quoted type defined in class scope.
+"""
+output = """
+annotations_forward_refs.py:41: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:42: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:43: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:44: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:45: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:46: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:47: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:48: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:49: error: Variable "annotations_forward_refs.var1" is not valid as a type  [valid-type]
+annotations_forward_refs.py:49: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
+annotations_forward_refs.py:50: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:51: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:52: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:53: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:54: error: Invalid type comment or annotation  [valid-type]
+annotations_forward_refs.py:55: error: Module "types" is not valid as a type  [valid-type]
+annotations_forward_refs.py:55: note: Perhaps you meant to use a protocol matching the module structure?
+annotations_forward_refs.py:80: error: Name "ClassF" is not defined  [name-defined]
+annotations_forward_refs.py:87: error: Function "annotations_forward_refs.ClassD.int" is not valid as a type  [valid-type]
+annotations_forward_refs.py:87: note: Perhaps you need "Callable[...]" or a callback protocol?
+annotations_forward_refs.py:89: error: Function "annotations_forward_refs.ClassD.int" is not valid as a type  [valid-type]
+annotations_forward_refs.py:89: note: Perhaps you need "Callable[...]" or a callback protocol?
+annotations_forward_refs.py:93: error: Expression is of type int?, not "int"  [assert-type]
+"""

conformance/results/mypy/annotations_generators.toml

@@ -0,0 +1,15 @@
+conformant = "Partial"
+notes = """
+Does not report incompatible Generator type in `yield from` statement.
+"""
+output = """
+annotations_generators.py:50: error: Missing return statement  [return]
+annotations_generators.py:53: error: Incompatible return value type (got "bool", expected "C")  [return-value]
+annotations_generators.py:56: error: Incompatible types in "yield" (actual type "int", expected type "A")  [misc]
+annotations_generators.py:65: error: Incompatible types in "yield" (actual type "int", expected type "A")  [misc]
+annotations_generators.py:74: error: Incompatible types in "yield" (actual type "B", expected type "A")  [misc]
+annotations_generators.py:85: error: The return type of a generator function should be "Generator" or one of its supertypes  [misc]
+annotations_generators.py:90: error: The return type of an async generator function should be "AsyncGenerator" or one of its supertypes  [misc]
+annotations_generators.py:117: error: Incompatible types in "yield from" (actual type "A", expected type "B")  [misc]
+annotations_generators.py:118: error: Incompatible types in "yield from" (actual type "int", expected type "B")  [misc]
+"""

conformance/results/mypy/annotations_methods.toml

@@ -0,0 +1,3 @@
+conformant = "Pass"
+output = """
+"""

conformance/results/mypy/annotations_typeexpr.toml

@@ -1,20 +1,22 @@
 conformant = "Pass"
 output = """
-annotations_typeexpr.py:77: error: Invalid type comment or annotation  [valid-type]
-annotations_typeexpr.py:77: note: Suggestion: use eval[...] instead of eval(...)
-annotations_typeexpr.py:78: error: Bracketed expression "[...]" is not valid as a type  [valid-type]
-annotations_typeexpr.py:79: error: Syntax error in type annotation  [syntax]
-annotations_typeexpr.py:79: note: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn)
-annotations_typeexpr.py:80: error: Invalid type comment or annotation  [valid-type]
-annotations_typeexpr.py:81: error: Invalid type comment or annotation  [valid-type]
-annotations_typeexpr.py:82: error: Invalid type comment or annotation  [valid-type]
-annotations_typeexpr.py:83: error: Invalid type comment or annotation  [valid-type]
-annotations_typeexpr.py:84: error: Invalid type comment or annotation  [valid-type]
-annotations_typeexpr.py:85: error: Variable "annotations_typeexpr.var1" is not valid as a type  [valid-type]
-annotations_typeexpr.py:85: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
-annotations_typeexpr.py:86: error: Invalid type: try using Literal[True] instead?  [valid-type]
-annotations_typeexpr.py:87: error: Invalid type: try using Literal[1] instead?  [valid-type]
-annotations_typeexpr.py:88: error: Invalid type: try using Literal[-1] instead?  [valid-type]
-annotations_typeexpr.py:89: error: Invalid type comment or annotation  [valid-type]
-annotations_typeexpr.py:90: error: Invalid type comment or annotation  [valid-type]
+annotations_typeexpr.py:88: error: Invalid type comment or annotation  [valid-type]
+annotations_typeexpr.py:88: note: Suggestion: use eval[...] instead of eval(...)
+annotations_typeexpr.py:89: error: Bracketed expression "[...]" is not valid as a type  [valid-type]
+annotations_typeexpr.py:90: error: Syntax error in type annotation  [syntax]
+annotations_typeexpr.py:90: note: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn)
+annotations_typeexpr.py:91: error: Invalid type comment or annotation  [valid-type]
+annotations_typeexpr.py:92: error: Invalid type comment or annotation  [valid-type]
+annotations_typeexpr.py:93: error: Invalid type comment or annotation  [valid-type]
+annotations_typeexpr.py:94: error: Invalid type comment or annotation  [valid-type]
+annotations_typeexpr.py:95: error: Invalid type comment or annotation  [valid-type]
+annotations_typeexpr.py:96: error: Variable "annotations_typeexpr.var1" is not valid as a type  [valid-type]
+annotations_typeexpr.py:96: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
+annotations_typeexpr.py:97: error: Invalid type: try using Literal[True] instead?  [valid-type]
+annotations_typeexpr.py:98: error: Invalid type: try using Literal[1] instead?  [valid-type]
+annotations_typeexpr.py:99: error: Invalid type: try using Literal[-1] instead?  [valid-type]
+annotations_typeexpr.py:100: error: Invalid type comment or annotation  [valid-type]
+annotations_typeexpr.py:101: error: Invalid type comment or annotation  [valid-type]
+annotations_typeexpr.py:102: error: Module "types" is not valid as a type  [valid-type]
+annotations_typeexpr.py:102: note: Perhaps you meant to use a protocol matching the module structure?
 """

conformance/results/mypy/classes_classvar.toml

@@ -0,0 +1,31 @@
+conformant = "Partial"
+notes = """
+Internal error if TypeVarTuple is used in ClassVar.
+Does not reject use of ParamSpec in ClassVar.
+Rejects ClassVar nested in Annotated.
+Does not reject use of ClassVar in TypeAlias definition.
+Does not infer type of ClassVar from assignment if no type is provided.
+"""
+output = """
+classes_classvar.py:36: error: ClassVar[...] must have at most one type argument  [valid-type]
+classes_classvar.py:37: error: Invalid type: try using Literal[3] instead?  [valid-type]
+classes_classvar.py:38: error: Name "var" is not defined  [name-defined]
+classes_classvar.py:43: error: ClassVar cannot contain type variables  [misc]
+classes_classvar.py:44: error: ClassVar cannot contain type variables  [misc]
+classes_classvar.py:50: error: Incompatible types in assignment (expression has type "dict[Never, Never]", variable has type "list[str]")  [assignment]
+classes_classvar.py:52: error: Name "Final" is not defined  [name-defined]
+classes_classvar.py:53: error: Invalid type: ClassVar nested inside other type  [valid-type]
+classes_classvar.py:59: error: Invalid type: ClassVar nested inside other type  [valid-type]
+classes_classvar.py:61: error: ClassVar can only be used for assignments in class body  [misc]
+classes_classvar.py:62: error: ClassVar can only be used for assignments in class body  [misc]
+classes_classvar.py:63: error: ClassVar can only be used for assignments in class body  [misc]
+classes_classvar.py:65: error: ClassVar can only be used for assignments in class body  [misc]
+classes_classvar.py:69: error: ClassVar can only be used for assignments in class body  [misc]
+classes_classvar.py:76: error: Expression is of type "Any", not "float"  [assert-type]
+classes_classvar.py:100: error: Cannot assign to class variable "stats" via instance  [misc]
+classes_classvar.py:129: error: Incompatible types in assignment (expression has type "ProtoAImpl", variable has type "ProtoA")  [assignment]
+classes_classvar.py:129: note: "ProtoAImpl" is missing following "ProtoA" protocol member:
+classes_classvar.py:129: note:     z
+classes_classvar.py:129: note: Protocol member ProtoA.x expected class variable, got instance variable
+classes_classvar.py:129: note: Protocol member ProtoA.y expected class variable, got instance variable
+"""

conformance/results/mypy/classes_override.toml

@@ -0,0 +1,12 @@
+conformant = "Partial"
+notes = """
+Does not handle case where parent class derives from Any.
+"""
+output = """
+classes_override.py:53: error: Method "method3" is marked as an override, but no base method was found with this name  [misc]
+classes_override.py:56: error: Method "method4" is marked as an override, but no base method was found with this name  [misc]
+classes_override.py:79: error: Method "static_method1" is marked as an override, but no base method was found with this name  [misc]
+classes_override.py:84: error: Method "class_method1" is marked as an override, but no base method was found with this name  [misc]
+classes_override.py:91: error: Method "property1" is marked as an override, but no base method was found with this name  [misc]
+classes_override.py:103: error: Method "method1" is marked as an override, but no base method was found with this name  [misc]
+"""

conformance/results/mypy/directives_assert_type.toml

@@ -0,0 +1,10 @@
+conformant = "Pass"
+output = """
+directives_assert_type.py:27: error: Expression is of type "int | str", not "int"  [assert-type]
+directives_assert_type.py:28: error: Expression is of type "Any", not "int"  [assert-type]
+directives_assert_type.py:30: error: "assert_type" expects 2 arguments  [misc]
+directives_assert_type.py:30: error: Too few arguments for "assert_type"  [call-arg]
+directives_assert_type.py:31: error: Expression is of type "Literal['']", not "int"  [assert-type]
+directives_assert_type.py:32: error: "assert_type" expects 2 arguments  [misc]
+directives_assert_type.py:32: error: Too many arguments for "assert_type"  [call-arg]
+"""

conformance/results/mypy/directives_cast.toml

@@ -0,0 +1,16 @@
+conformant = "Pass"
+output = """
+directives_cast.py:15: error: "cast" expects 2 arguments  [misc]
+directives_cast.py:15: error: All overload variants of "cast" require at least one argument  [call-overload]
+directives_cast.py:15: note: Possible overload variants:
+directives_cast.py:15: note:     def [_T] cast(typ: type[_T], val: Any) -> _T
+directives_cast.py:15: note:     def cast(typ: str, val: Any) -> Any
+directives_cast.py:15: note:     def cast(typ: object, val: Any) -> Any
+directives_cast.py:16: error: Invalid type: try using Literal[1] instead?  [valid-type]
+directives_cast.py:17: error: "cast" expects 2 arguments  [misc]
+directives_cast.py:17: error: No overload variant of "cast" matches argument types "Any", "str", "str"  [call-overload]
+directives_cast.py:17: note: Possible overload variants:
+directives_cast.py:17: note:     def [_T] cast(typ: type[_T], val: Any) -> _T
+directives_cast.py:17: note:     def cast(typ: str, val: Any) -> Any
+directives_cast.py:17: note:     def cast(typ: object, val: Any) -> Any
+"""

conformance/results/mypy/directives_no_type_check.toml

@@ -0,0 +1,3 @@
+conformant = "Pass"
+output = """
+"""

conformance/results/mypy/directives_reveal_type.toml

@@ -0,0 +1,11 @@
+conformant = "Pass"
+output = """
+directives_reveal_type.py:14: note: Revealed type is "Union[builtins.int, builtins.str]"
+directives_reveal_type.py:15: note: Revealed type is "builtins.list[builtins.int]"
+directives_reveal_type.py:16: note: Revealed type is "Any"
+directives_reveal_type.py:17: note: Revealed type is "directives_reveal_type.ForwardReference"
+directives_reveal_type.py:19: error: "reveal_type" expects 1 argument  [misc]
+directives_reveal_type.py:19: error: Too few arguments for "reveal_type"  [call-arg]
+directives_reveal_type.py:20: error: "reveal_type" expects 1 argument  [misc]
+directives_reveal_type.py:20: error: Too many arguments for "reveal_type"  [call-arg]
+"""

conformance/results/mypy/directives_type_checking.toml

@@ -0,0 +1,3 @@
+conformant = "Pass"
+output = """
+"""

conformance/results/mypy/directives_type_ignore.toml

@@ -0,0 +1,10 @@
+conformant = "Partial"
+notes = """
+Does not honor "# type: ignore" comment if comment includes additional text.
+"""
+output = """
+directives_type_ignore.py:11: error: Invalid "type: ignore" comment  [syntax]
+directives_type_ignore.py:11: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
+directives_type_ignore.py:14: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
+directives_type_ignore.py:14: note: Error code "assignment" not covered by "type: ignore" comment
+"""

conformance/results/mypy/directives_version_platform.toml

@@ -0,0 +1,11 @@
+conformant = "Pass"
+notes = """
+Does not understand three-element form of sys.version checks.
+Does not understand os.name checks.
+"""
+output = """
+directives_version_platform.py:19: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
+directives_version_platform.py:27: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
+directives_version_platform.py:40: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
+directives_version_platform.py:45: error: Incompatible types in assignment (expression has type "str", variable has type "int")  [assignment]
+"""

conformance/results/mypy/overloads_basic.toml

@@ -0,0 +1,9 @@
+conformant = "Pass"
+output = """
+overloads_basic.py:35: error: No overload variant of "__getitem__" of "Bytes" matches argument type "str"  [call-overload]
+overloads_basic.py:35: note: Possible overload variants:
+overloads_basic.py:35: note:     def __getitem__(self, int, /) -> int
+overloads_basic.py:35: note:     def __getitem__(self, slice, /) -> bytes
+overloads_basic.py:59: error: Single overload definition, multiple required  [misc]
+overloads_basic.py:69: error: An overloaded function outside a stub file must have an implementation  [no-overload-impl]
+"""

conformance/results/mypy/protocols_class_objects.toml

@@ -0,0 +1,23 @@
+conformant = "Pass"
+output = """
+protocols_class_objects.py:29: error: Only concrete class can be given where "type[Proto]" is expected  [type-abstract]
+protocols_class_objects.py:34: error: Can only assign concrete classes to a variable of type "type[Proto]"  [type-abstract]
+protocols_class_objects.py:58: error: Incompatible types in assignment (expression has type "type[ConcreteA]", variable has type "ProtoA1")  [assignment]
+protocols_class_objects.py:58: note: Following member(s) of "ConcreteA" have conflicts:
+protocols_class_objects.py:58: note:     Expected:
+protocols_class_objects.py:58: note:         def method1(x: int) -> int
+protocols_class_objects.py:58: note:     Got:
+protocols_class_objects.py:58: note:         def method1(self: ConcreteA, x: int) -> int
+protocols_class_objects.py:74: error: Incompatible types in assignment (expression has type "type[ConcreteB]", variable has type "ProtoB1")  [assignment]
+protocols_class_objects.py:74: note: Following member(s) of "ConcreteB" have conflicts:
+protocols_class_objects.py:74: note:     prop1: expected "int", got "Callable[[ConcreteB], int]"
+protocols_class_objects.py:74: note: Only class variables allowed for class object access on protocols, prop1 is an instance variable of "ConcreteB"
+protocols_class_objects.py:101: error: Incompatible types in assignment (expression has type "type[ConcreteC1]", variable has type "ProtoC1")  [assignment]
+protocols_class_objects.py:101: note: ClassVar protocol member ProtoC1.attr1 can never be matched by a class object
+protocols_class_objects.py:103: error: Incompatible types in assignment (expression has type "type[ConcreteC2]", variable has type "ProtoC1")  [assignment]
+protocols_class_objects.py:103: note: Only class variables allowed for class object access on protocols, attr1 is an instance variable of "ConcreteC2"
+protocols_class_objects.py:103: note: ClassVar protocol member ProtoC1.attr1 can never be matched by a class object
+protocols_class_objects.py:104: error: Incompatible types in assignment (expression has type "type[ConcreteC2]", variable has type "ProtoC2")  [assignment]
+protocols_class_objects.py:104: note: Only class variables allowed for class object access on protocols, attr1 is an instance variable of "ConcreteC2"
+protocols_class_objects.py:105: error: Incompatible types in assignment (expression has type "type[ConcreteC3]", variable has type "ProtoC1")  [assignment]
+"""