From ad0e183b0df7cc3dd94d9e1cd6f5710859beda96 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 30 Oct 2023 12:14:00 +0000 Subject: [PATCH] Enable Unpack/TypeVarTuple support (#16354) Fixes https://github.com/python/mypy/issues/12280 Fixes https://github.com/python/mypy/issues/14697 In this PR: * Enable `TypeVarTuple` and `Unpack` features. * Delete the old blanket `--enable-incomplete-features` flag that was deprecated a year ago. * Switch couple corner cases to `PreciseTupleTypes` feature. * Add the draft docs about the new feature. * Handle a previously unhandled case where variadic tuple appears in string formatting (discovered on mypy self-check, where `PreciseTupleTypes` is already enabled). --------- Co-authored-by: Jelle Zijlstra --- docs/source/command_line.rst | 52 +++++++++++++++++++++++++ mypy/checkexpr.py | 8 ++-- mypy/checkstrformat.py | 19 +++++++++ mypy/main.py | 17 +++----- mypy/options.py | 6 +-- mypy/semanal.py | 5 +-- mypy/test/testcheck.py | 3 -- mypy/test/testfinegrained.py | 3 +- mypy/test/testsemanal.py | 3 +- mypy/test/testtransform.py | 2 - mypy/typeanal.py | 4 +- test-data/unit/check-flags.test | 12 ------ test-data/unit/check-tuples.test | 16 ++++++++ test-data/unit/check-typevar-tuple.test | 3 ++ test-data/unit/cmdline.test | 18 +++++---- 15 files changed, 116 insertions(+), 55 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 5db118334519..a810c35cb77f 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -991,6 +991,58 @@ format into the specified directory. library or specify mypy installation with the setuptools extra ``mypy[reports]``. + +Enabling incomplete/experimental features +***************************************** + +.. option:: --enable-incomplete-feature FEATURE + + Some features may require several mypy releases to implement, for example + due to their complexity, potential for backwards incompatibility, or + ambiguous semantics that would benefit from feedback from the community. + You can enable such features for early preview using this flag. Note that + it is not guaranteed that all features will be ultimately enabled by + default. In *rare cases* we may decide to not go ahead with certain + features. + +List of currently incomplete/experimental features: + +* ``PreciseTupleTypes``: this feature will infer more precise tuple types in + various scenarios. Before variadic types were added to the Python type system + by :pep:`646`, it was impossible to express a type like "a tuple with + at least two integers". The best type available was ``tuple[int, ...]``. + Therefore, mypy applied very lenient checking for variable-length tuples. + Now this type can be expressed as ``tuple[int, int, *tuple[int, ...]]``. + For such more precise types (when explicitly *defined* by a user) mypy, + for example, warns about unsafe index access, and generally handles them + in a type-safe manner. However, to avoid problems in existing code, mypy + does not *infer* these precise types when it technically can. Here are + notable examples where ``PreciseTupleTypes`` infers more precise types: + + .. code-block:: python + + numbers: tuple[int, ...] + + more_numbers = (1, *numbers, 1) + reveal_type(more_numbers) + # Without PreciseTupleTypes: tuple[int, ...] + # With PreciseTupleTypes: tuple[int, *tuple[int, ...], int] + + other_numbers = (1, 1) + numbers + reveal_type(other_numbers) + # Without PreciseTupleTypes: tuple[int, ...] + # With PreciseTupleTypes: tuple[int, int, *tuple[int, ...]] + + if len(numbers) > 2: + reveal_type(numbers) + # Without PreciseTupleTypes: tuple[int, ...] + # With PreciseTupleTypes: tuple[int, int, int, *tuple[int, ...]] + else: + reveal_type(numbers) + # Without PreciseTupleTypes: tuple[int, ...] + # With PreciseTupleTypes: tuple[()] | tuple[int] | tuple[int, int] + + Miscellaneous ************* diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index df6000050986..0207c245b1f9 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -97,7 +97,7 @@ YieldExpr, YieldFromExpr, ) -from mypy.options import TYPE_VAR_TUPLE +from mypy.options import PRECISE_TUPLE_TYPES from mypy.plugin import ( FunctionContext, FunctionSigContext, @@ -3377,7 +3377,7 @@ def visit_op_expr(self, e: OpExpr) -> Type: ): return self.concat_tuples(proper_left_type, proper_right_type) elif ( - TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature + PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature and isinstance(proper_right_type, Instance) and self.chk.type_is_iterable(proper_right_type) ): @@ -3411,7 +3411,7 @@ def visit_op_expr(self, e: OpExpr) -> Type: if is_named_instance(proper_right_type, "builtins.dict"): use_reverse = USE_REVERSE_NEVER - if TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature: + if PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature: # Handle tuple[X, ...] + tuple[Y, Z] = tuple[*tuple[X, ...], Y, Z]. if ( e.op == "+" @@ -4988,7 +4988,7 @@ def visit_tuple_expr(self, e: TupleExpr) -> Type: j += len(tt.items) else: if ( - TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature + PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature and not seen_unpack_in_items ): # Handle (x, *y, z), where y is e.g. tuple[Y, ...]. diff --git a/mypy/checkstrformat.py b/mypy/checkstrformat.py index eeb9e7633756..39d44e84a9c1 100644 --- a/mypy/checkstrformat.py +++ b/mypy/checkstrformat.py @@ -47,8 +47,11 @@ TupleType, Type, TypeOfAny, + TypeVarTupleType, TypeVarType, UnionType, + UnpackType, + find_unpack_in_list, get_proper_type, get_proper_types, ) @@ -728,6 +731,22 @@ def check_simple_str_interpolation( rep_types: list[Type] = [] if isinstance(rhs_type, TupleType): rep_types = rhs_type.items + unpack_index = find_unpack_in_list(rep_types) + if unpack_index is not None: + # TODO: we should probably warn about potentially short tuple. + # However, without special-casing for tuple(f(i) for in other_tuple) + # this causes false positive on mypy self-check in report.py. + extras = max(0, len(checkers) - len(rep_types) + 1) + unpacked = rep_types[unpack_index] + assert isinstance(unpacked, UnpackType) + unpacked = get_proper_type(unpacked.type) + if isinstance(unpacked, TypeVarTupleType): + unpacked = get_proper_type(unpacked.upper_bound) + assert ( + isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple" + ) + unpack_items = [unpacked.args[0]] * extras + rep_types = rep_types[:unpack_index] + unpack_items + rep_types[unpack_index + 1 :] elif isinstance(rhs_type, AnyType): return elif isinstance(rhs_type, Instance) and rhs_type.type.fullname == "builtins.tuple": diff --git a/mypy/main.py b/mypy/main.py index 43ab761072ca..1aede530c33e 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -22,7 +22,7 @@ from mypy.find_sources import InvalidSourceList, create_source_list from mypy.fscache import FileSystemCache from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths, get_search_dirs, mypy_path -from mypy.options import INCOMPLETE_FEATURES, BuildType, Options +from mypy.options import COMPLETE_FEATURES, INCOMPLETE_FEATURES, BuildType, Options from mypy.split_namespace import SplitNamespace from mypy.version import __version__ @@ -1151,10 +1151,7 @@ def add_invertible_flag( # --debug-serialize will run tree.serialize() even if cache generation is disabled. # Useful for mypy_primer to detect serialize errors earlier. parser.add_argument("--debug-serialize", action="store_true", help=argparse.SUPPRESS) - # This one is deprecated, but we will keep it for few releases. - parser.add_argument( - "--enable-incomplete-features", action="store_true", help=argparse.SUPPRESS - ) + parser.add_argument( "--disable-bytearray-promotion", action="store_true", help=argparse.SUPPRESS ) @@ -1334,14 +1331,10 @@ def set_strict_flags() -> None: # Validate incomplete features. for feature in options.enable_incomplete_feature: - if feature not in INCOMPLETE_FEATURES: + if feature not in INCOMPLETE_FEATURES | COMPLETE_FEATURES: parser.error(f"Unknown incomplete feature: {feature}") - if options.enable_incomplete_features: - print( - "Warning: --enable-incomplete-features is deprecated, use" - " --enable-incomplete-feature=FEATURE instead" - ) - options.enable_incomplete_feature = list(INCOMPLETE_FEATURES) + if feature in COMPLETE_FEATURES: + print(f"Warning: {feature} is already enabled by default") # Compute absolute path for custom typeshed (if present). if options.custom_typeshed_dir is not None: diff --git a/mypy/options.py b/mypy/options.py index 31d5d584f897..8bb20dbd4410 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -69,11 +69,12 @@ class BuildType: } ) - {"debug_cache"} -# Features that are currently incomplete/experimental +# Features that are currently (or were recently) incomplete/experimental TYPE_VAR_TUPLE: Final = "TypeVarTuple" UNPACK: Final = "Unpack" PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes" -INCOMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK, PRECISE_TUPLE_TYPES)) +INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES,)) +COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK)) class Options: @@ -307,7 +308,6 @@ def __init__(self) -> None: self.dump_type_stats = False self.dump_inference_stats = False self.dump_build_stats = False - self.enable_incomplete_features = False # deprecated self.enable_incomplete_feature: list[str] = [] self.timing_stats: str | None = None self.line_checking_stats: str | None = None diff --git a/mypy/semanal.py b/mypy/semanal.py index bd24c48ed24f..6f322af816ea 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -179,7 +179,7 @@ type_aliases_source_versions, typing_extensions_aliases, ) -from mypy.options import TYPE_VAR_TUPLE, Options +from mypy.options import Options from mypy.patterns import ( AsPattern, ClassPattern, @@ -4417,9 +4417,6 @@ def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool: else: self.fail(f'Unexpected keyword argument "{param_name}" for "TypeVarTuple"', s) - if not self.incomplete_feature_enabled(TYPE_VAR_TUPLE, s): - return False - name = self.extract_typevarlike_name(s, call) if name is None: return False diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 591421465a97..3ad97ced61f2 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -10,7 +10,6 @@ from mypy.build import Graph from mypy.errors import CompileError from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths -from mypy.options import TYPE_VAR_TUPLE, UNPACK from mypy.test.config import test_data_prefix, test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite, FileOperation, module_from_path from mypy.test.helpers import ( @@ -125,8 +124,6 @@ def run_case_once( # Parse options after moving files (in case mypy.ini is being moved). options = parse_options(original_program_text, testcase, incremental_step) options.use_builtins_fixtures = True - if not testcase.name.endswith("_no_incomplete"): - options.enable_incomplete_feature += [TYPE_VAR_TUPLE, UNPACK] options.show_traceback = True # Enable some options automatically based on test file name. diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index c517c54286d7..953f91a60df7 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -28,7 +28,7 @@ from mypy.errors import CompileError from mypy.find_sources import create_source_list from mypy.modulefinder import BuildSource -from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options +from mypy.options import Options from mypy.server.mergecheck import check_consistency from mypy.server.update import sort_messages_preserving_file_order from mypy.test.config import test_temp_dir @@ -149,7 +149,6 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo options.use_fine_grained_cache = self.use_cache and not build_cache options.cache_fine_grained = self.use_cache options.local_partial_types = True - options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK] # Treat empty bodies safely for these test cases. options.allow_empty_bodies = not testcase.name.endswith("_no_empty") if re.search("flags:.*--follow-imports", source) is None: diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py index 3455f41aa20a..cdecc4739168 100644 --- a/mypy/test/testsemanal.py +++ b/mypy/test/testsemanal.py @@ -10,7 +10,7 @@ from mypy.errors import CompileError from mypy.modulefinder import BuildSource from mypy.nodes import TypeInfo -from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options +from mypy.options import Options from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import ( @@ -45,7 +45,6 @@ def get_semanal_options(program_text: str, testcase: DataDrivenTestCase) -> Opti options.semantic_analysis_only = True options.show_traceback = True options.python_version = PYTHON3_VERSION - options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK] options.force_uppercase_builtins = True return options diff --git a/mypy/test/testtransform.py b/mypy/test/testtransform.py index ba9fe8668fb4..9388dca02c7a 100644 --- a/mypy/test/testtransform.py +++ b/mypy/test/testtransform.py @@ -5,7 +5,6 @@ from mypy import build from mypy.errors import CompileError from mypy.modulefinder import BuildSource -from mypy.options import TYPE_VAR_TUPLE, UNPACK from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite from mypy.test.helpers import assert_string_arrays_equal, normalize_error_messages, parse_options @@ -38,7 +37,6 @@ def test_transform(testcase: DataDrivenTestCase) -> None: options = parse_options(src, testcase, 1) options.use_builtins_fixtures = True options.semantic_analysis_only = True - options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK] options.show_traceback = True options.force_uppercase_builtins = True result = build.build( diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 03579404aac9..d238a452e7a9 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -35,7 +35,7 @@ check_arg_names, get_nongen_builtins, ) -from mypy.options import UNPACK, Options +from mypy.options import Options from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface from mypy.semanal_shared import SemanticAnalyzerCoreInterface, paramspec_args, paramspec_kwargs from mypy.tvar_scope import TypeVarLikeScope @@ -664,8 +664,6 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ # In most contexts, TypeGuard[...] acts as an alias for bool (ignoring its args) return self.named_type("builtins.bool") elif fullname in ("typing.Unpack", "typing_extensions.Unpack"): - if not self.api.incomplete_feature_enabled(UNPACK, t): - return AnyType(TypeOfAny.from_error) if len(t.args) != 1: self.fail("Unpack[...] requires exactly one type argument", t) return AnyType(TypeOfAny.from_error) diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test index 546d02a07ad0..04adaca317c1 100644 --- a/test-data/unit/check-flags.test +++ b/test-data/unit/check-flags.test @@ -2190,18 +2190,6 @@ x: int = "" # E: Incompatible types in assignment (expression has type "str", v # flags: --hide-error-codes x: int = "" # E: Incompatible types in assignment (expression has type "str", variable has type "int") -[case testTypeVarTupleDisabled_no_incomplete] -from typing_extensions import TypeVarTuple -Ts = TypeVarTuple("Ts") # E: "TypeVarTuple" support is experimental, use --enable-incomplete-feature=TypeVarTuple to enable -[builtins fixtures/tuple.pyi] - -[case testTypeVarTupleEnabled_no_incomplete] -# flags: --enable-incomplete-feature=TypeVarTuple -from typing_extensions import TypeVarTuple -Ts = TypeVarTuple("Ts") # OK -[builtins fixtures/tuple.pyi] - - [case testDisableBytearrayPromotion] # flags: --disable-bytearray-promotion def f(x: bytes) -> None: ... diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index 7070ead43746..4f468b59fc3f 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -1100,12 +1100,28 @@ reveal_type(b) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtin [case testTupleWithStarExpr2] a = [1] b = (0, *a) +reveal_type(b) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/tuple.pyi] + +[case testTupleWithStarExpr2Precise] +# flags: --enable-incomplete-feature=PreciseTupleTypes +a = [1] +b = (0, *a) reveal_type(b) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]]]" [builtins fixtures/tuple.pyi] [case testTupleWithStarExpr3] a = [''] b = (0, *a) +reveal_type(b) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +c = (*a, '') +reveal_type(c) # N: Revealed type is "builtins.tuple[builtins.str, ...]" +[builtins fixtures/tuple.pyi] + +[case testTupleWithStarExpr3Precise] +# flags: --enable-incomplete-feature=PreciseTupleTypes +a = [''] +b = (0, *a) reveal_type(b) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.str, ...]]]" c = (*a, '') reveal_type(c) # N: Revealed type is "Tuple[Unpack[builtins.tuple[builtins.str, ...]], builtins.str]" diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 7b8a22313b36..a51b535a873c 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -1653,6 +1653,7 @@ def foo(arg: Tuple[int, Unpack[Ts], str]) -> None: [builtins fixtures/tuple.pyi] [case testPackingVariadicTuplesHomogeneous] +# flags: --enable-incomplete-feature=PreciseTupleTypes from typing import Tuple from typing_extensions import Unpack @@ -1689,6 +1690,7 @@ def foo(arg: Tuple[int, Unpack[Ts], str]) -> None: [builtins fixtures/isinstancelist.pyi] [case testVariadicTupleInTupleContext] +# flags: --enable-incomplete-feature=PreciseTupleTypes from typing import Tuple, Optional from typing_extensions import TypeVarTuple, Unpack @@ -1701,6 +1703,7 @@ vt2 = 1, *test(), 2 # E: Need type annotation for "vt2" [builtins fixtures/tuple.pyi] [case testVariadicTupleConcatenation] +# flags: --enable-incomplete-feature=PreciseTupleTypes from typing import Tuple from typing_extensions import TypeVarTuple, Unpack diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 91242eb62fcf..f286f4781ed5 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -1421,14 +1421,6 @@ b \d+ b\.c \d+ .* -[case testCmdlineEnableIncompleteFeatures] -# cmd: mypy --enable-incomplete-features a.py -[file a.py] -pass -[out] -Warning: --enable-incomplete-features is deprecated, use --enable-incomplete-feature=FEATURE instead -== Return code: 0 - [case testShadowTypingModuleEarlyLoad] # cmd: mypy dir [file dir/__init__.py] @@ -1585,3 +1577,13 @@ disable_error_code = always_true = MY_VAR, [out] + +[case testTypeVarTupleUnpackEnabled] +# cmd: mypy --enable-incomplete-feature=TypeVarTuple --enable-incomplete-feature=Unpack a.py +[file a.py] +from typing_extensions import TypeVarTuple +Ts = TypeVarTuple("Ts") +[out] +Warning: TypeVarTuple is already enabled by default +Warning: Unpack is already enabled by default +== Return code: 0