diff --git a/crates/ty_python_semantic/src/semantic_index/builder.rs b/crates/ty_python_semantic/src/semantic_index/builder.rs index 17fa4147f0625..71645d77565e8 100644 --- a/crates/ty_python_semantic/src/semantic_index/builder.rs +++ b/crates/ty_python_semantic/src/semantic_index/builder.rs @@ -56,8 +56,8 @@ use crate::semantic_index::{ get_loop_header, }; use crate::semantic_model::HasTrackedScope; -use crate::types::PossiblyNarrowedPlaces; -use crate::unpack::{EvaluationMode, Unpack, UnpackKind, UnpackPosition, UnpackValue}; +use crate::types::{EvaluationMode, PossiblyNarrowedPlaces}; +use crate::unpack::{Unpack, UnpackKind, UnpackPosition, UnpackValue}; use crate::{Db, Program}; mod except_handlers; diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index e1304f3260569..0963ad40de8ca 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -11,14 +11,13 @@ use std::time::Duration; use bitflags::bitflags; use call::{CallDunderError, CallError, CallErrorKind}; use context::InferContext; -use diagnostic::{INVALID_CONTEXT_MANAGER, NOT_ITERABLE}; use ruff_db::Instant; -use ruff_db::diagnostic::{Annotation, Diagnostic, Span, SubDiagnostic, SubDiagnosticSeverity}; +use ruff_db::diagnostic::{Annotation, Diagnostic, Span}; use ruff_db::files::File; use ruff_db::parsed::parsed_module; use ruff_python_ast as ast; use ruff_python_ast::name::Name; -use ruff_text_size::{Ranged, TextRange}; +use ruff_text_size::Ranged; use smallvec::{SmallVec, smallvec_inline}; use ty_module_resolver::{KnownModule, Module, ModuleName, resolve_module}; @@ -57,7 +56,7 @@ use crate::types::constraints::{ ConstraintSet, ConstraintSetBuilder, IteratorConstraintsExtension, }; use crate::types::context::{LintDiagnosticGuard, LintDiagnosticGuardBuilder}; -use crate::types::diagnostic::{INVALID_AWAIT, INVALID_TYPE_FORM, UNSUPPORTED_BOOL_CONVERSION}; +use crate::types::diagnostic::{INVALID_AWAIT, INVALID_TYPE_FORM}; pub use crate::types::display::{DisplaySettings, TypeDetail, TypeDisplayDetails}; use crate::types::enums::{enum_metadata, is_single_member_enum}; use crate::types::function::{ @@ -78,13 +77,11 @@ use crate::types::newtype::NewType; pub(crate) use crate::types::signatures::{Parameter, Parameters}; use crate::types::signatures::{ParameterForm, walk_signature}; use crate::types::special_form::TypeQualifier; -use crate::types::tuple::{Tuple, TupleSpec, TupleSpecBuilder}; -use crate::types::typed_dict::TypedDictField; +use crate::types::tuple::{Tuple, TupleSpec}; pub(crate) use crate::types::typed_dict::{TypedDictParams, TypedDictType, walk_typed_dict_type}; pub use crate::types::variance::TypeVarVariance; use crate::types::variance::VarianceInferable; use crate::types::visitor::any_over_type; -use crate::unpack::EvaluationMode; use crate::{Db, FxOrderSet, Program}; pub use class::KnownClass; pub(crate) use class::{ClassLiteral, ClassType, GenericAlias, StaticClassLiteral}; @@ -95,12 +92,14 @@ pub(crate) use literal::{ }; pub use special_form::SpecialFormType; +mod bool; mod bound_super; mod call; mod class; mod class_base; mod constraints; mod context; +mod context_manager; mod cyclic; mod diagnostic; mod display; @@ -110,6 +109,7 @@ mod generics; pub mod ide_support; mod infer; mod instance; +mod iteration; mod known_instance; pub mod list_members; mod literal; @@ -232,11 +232,6 @@ pub(crate) type FindLegacyTypeVarsVisitor<'db> = CycleDetector = - CycleDetector, Result>>; -pub(crate) struct TryBool; - /// A [`CycleDetector`] that is used in `visit_specialization` methods. pub(crate) type SpecializationVisitor<'db> = CycleDetector, ()>; pub(crate) struct VisitSpecialization; @@ -3527,310 +3522,6 @@ impl<'db> Type<'db> { } } - /// Resolves the boolean value of the type and falls back to [`Truthiness::Ambiguous`] if the type doesn't implement `__bool__` correctly. - /// - /// This method should only be used outside type checking or when evaluating if a type - /// is truthy or falsy in a context where Python doesn't make an implicit `bool` call. - /// Use [`try_bool`](Self::try_bool) for type checking or implicit `bool` calls. - pub(crate) fn bool(&self, db: &'db dyn Db) -> Truthiness { - self.try_bool_impl(db, true, &TryBoolVisitor::new(Ok(Truthiness::Ambiguous))) - .unwrap_or_else(|err| err.fallback_truthiness()) - } - - /// Resolves the boolean value of a type. - /// - /// This is used to determine the value that would be returned - /// when `bool(x)` is called on an object `x`. - /// - /// Returns an error if the type doesn't implement `__bool__` correctly. - pub(crate) fn try_bool(&self, db: &'db dyn Db) -> Result> { - self.try_bool_impl(db, false, &TryBoolVisitor::new(Ok(Truthiness::Ambiguous))) - } - - /// Resolves the boolean value of a type. - /// - /// Setting `allow_short_circuit` to `true` allows the implementation to - /// early return if the bool value of any union variant is `Truthiness::Ambiguous`. - /// Early returning shows a 1-2% perf improvement on our benchmarks because - /// `bool` (which doesn't care about errors) is used heavily when evaluating statically known branches. - /// - /// An alternative to this flag is to implement a trait similar to Rust's `Try` trait. - /// The advantage of that is that it would allow collecting the errors as well. However, - /// it is significantly more complex and duplicating the logic into `bool` without the error - /// handling didn't show any significant performance difference to when using the `allow_short_circuit` flag. - #[inline] - fn try_bool_impl( - &self, - db: &'db dyn Db, - allow_short_circuit: bool, - visitor: &TryBoolVisitor<'db>, - ) -> Result> { - let type_to_truthiness = |ty: Type<'db>| { - match ty.as_literal_value_kind() { - Some(LiteralValueTypeKind::Bool(bool_val)) => Truthiness::from(bool_val), - Some(LiteralValueTypeKind::Int(int_val)) => Truthiness::from(int_val.as_i64() != 0), - // anything else is handled lower down - _ => Truthiness::Ambiguous, - } - }; - - let try_dunders = || { - match self.try_call_dunder( - db, - "__bool__", - CallArguments::none(), - TypeContext::default(), - ) { - Ok(outcome) => { - let return_type = outcome.return_type(db); - if !return_type.is_assignable_to(db, KnownClass::Bool.to_instance(db)) { - // The type has a `__bool__` method, but it doesn't return a - // boolean. - return Err(BoolError::IncorrectReturnType { - return_type, - not_boolable_type: *self, - }); - } - Ok(type_to_truthiness(return_type)) - } - - Err(CallDunderError::PossiblyUnbound(outcome)) => { - let return_type = outcome.return_type(db); - if !return_type.is_assignable_to(db, KnownClass::Bool.to_instance(db)) { - // The type has a `__bool__` method, but it doesn't return a - // boolean. - return Err(BoolError::IncorrectReturnType { - return_type: outcome.return_type(db), - not_boolable_type: *self, - }); - } - - // Don't trust possibly missing `__bool__` method. - Ok(Truthiness::Ambiguous) - } - - Err(CallDunderError::MethodNotAvailable) => { - // We only consider `__len__` for tuples and `@final` types, - // since `__bool__` takes precedence - // and a subclass could add a `__bool__` method. - // - // TODO: with regards to tuple types, we intend to emit a diagnostic - // if a tuple subclass defines a `__bool__` method with a return type - // that is inconsistent with the tuple's length. Otherwise, the special - // handling for tuples here isn't sound. - if let Some(instance) = self.as_nominal_instance() { - if let Some(tuple_spec) = instance.tuple_spec(db) { - Ok(tuple_spec.truthiness()) - } else if instance.class(db).is_final(db) { - match self.try_call_dunder( - db, - "__len__", - CallArguments::none(), - TypeContext::default(), - ) { - Ok(outcome) => { - let return_type = outcome.return_type(db); - if return_type.is_assignable_to( - db, - KnownClass::SupportsIndex.to_instance(db), - ) { - Ok(type_to_truthiness(return_type)) - } else { - // TODO: should report a diagnostic similar to if return type of `__bool__` - // is not assignable to `bool` - Ok(Truthiness::Ambiguous) - } - } - // if a `@final` type does not define `__bool__` or `__len__`, it is always truthy - Err(CallDunderError::MethodNotAvailable) => { - Ok(Truthiness::AlwaysTrue) - } - // TODO: errors during a `__len__` call (if `__len__` exists) should be reported - // as diagnostics similar to errors during a `__bool__` call (when `__bool__` exists) - Err(_) => Ok(Truthiness::Ambiguous), - } - } else { - Ok(Truthiness::Ambiguous) - } - } else { - Ok(Truthiness::Ambiguous) - } - } - - Err(CallDunderError::CallError(CallErrorKind::BindingError, bindings)) => { - Err(BoolError::IncorrectArguments { - truthiness: type_to_truthiness(bindings.return_type(db)), - not_boolable_type: *self, - }) - } - - Err(CallDunderError::CallError(CallErrorKind::NotCallable, _)) => { - Err(BoolError::NotCallable { - not_boolable_type: *self, - }) - } - - Err(CallDunderError::CallError(CallErrorKind::PossiblyNotCallable, _)) => { - Err(BoolError::Other { - not_boolable_type: *self, - }) - } - } - }; - - let try_union = |union: UnionType<'db>| { - let mut truthiness = None; - let mut all_not_callable = true; - let mut has_errors = false; - - for element in union.elements(db) { - let element_truthiness = - match element.try_bool_impl(db, allow_short_circuit, visitor) { - Ok(truthiness) => truthiness, - Err(err) => { - has_errors = true; - all_not_callable &= matches!(err, BoolError::NotCallable { .. }); - err.fallback_truthiness() - } - }; - - truthiness.get_or_insert(element_truthiness); - - if Some(element_truthiness) != truthiness { - truthiness = Some(Truthiness::Ambiguous); - - if allow_short_circuit { - return Ok(Truthiness::Ambiguous); - } - } - } - - if has_errors { - if all_not_callable { - return Err(BoolError::NotCallable { - not_boolable_type: *self, - }); - } - return Err(BoolError::Union { - union, - truthiness: truthiness.unwrap_or(Truthiness::Ambiguous), - }); - } - Ok(truthiness.unwrap_or(Truthiness::Ambiguous)) - }; - - let truthiness = match self { - Type::Dynamic(_) - | Type::Never - | Type::Callable(_) - | Type::TypeIs(_) - | Type::TypeGuard(_) => Truthiness::Ambiguous, - - Type::TypedDict(td) => { - if td.items(db).values().any(TypedDictField::is_required) { - Truthiness::AlwaysTrue - } else { - // We can potentially infer empty typeddicts as always falsy if they're `closed=True`, - // but as of 22-01-26 we don't yet support PEP 728. - Truthiness::Ambiguous - } - } - - Type::KnownInstance(KnownInstanceType::ConstraintSet(tracked_set)) => { - let constraints = ConstraintSetBuilder::new(); - let tracked_set = constraints.load(tracked_set.constraints(db)); - Truthiness::from(tracked_set.is_always_satisfied(db)) - } - - Type::FunctionLiteral(_) - | Type::BoundMethod(_) - | Type::WrapperDescriptor(_) - | Type::KnownBoundMethod(_) - | Type::DataclassDecorator(_) - | Type::DataclassTransformer(_) - | Type::ModuleLiteral(_) - | Type::PropertyInstance(_) - | Type::BoundSuper(_) - | Type::KnownInstance(_) - | Type::SpecialForm(_) - | Type::AlwaysTruthy => Truthiness::AlwaysTrue, - - Type::AlwaysFalsy => Truthiness::AlwaysFalse, - - Type::ClassLiteral(class) => { - class - .metaclass_instance_type(db) - .try_bool_impl(db, allow_short_circuit, visitor)? - } - Type::GenericAlias(alias) => ClassType::from(*alias) - .metaclass_instance_type(db) - .try_bool_impl(db, allow_short_circuit, visitor)?, - - Type::SubclassOf(subclass_of_ty) => { - match subclass_of_ty.subclass_of().with_transposed_type_var(db) { - SubclassOfInner::Dynamic(_) => Truthiness::Ambiguous, - SubclassOfInner::Class(class) => { - Type::from(class).try_bool_impl(db, allow_short_circuit, visitor)? - } - SubclassOfInner::TypeVar(bound_typevar) => Type::TypeVar(bound_typevar) - .try_bool_impl(db, allow_short_circuit, visitor)?, - } - } - - Type::TypeVar(bound_typevar) => { - match bound_typevar.typevar(db).bound_or_constraints(db) { - None => Truthiness::Ambiguous, - Some(TypeVarBoundOrConstraints::UpperBound(bound)) => { - bound.try_bool_impl(db, allow_short_circuit, visitor)? - } - Some(TypeVarBoundOrConstraints::Constraints(constraints)) => constraints - .as_type(db) - .try_bool_impl(db, allow_short_circuit, visitor)?, - } - } - - Type::NominalInstance(instance) => instance - .known_class(db) - .and_then(KnownClass::bool) - .map(Ok) - .unwrap_or_else(try_dunders)?, - - Type::ProtocolInstance(_) => try_dunders()?, - - Type::Union(union) => try_union(*union)?, - - Type::Intersection(_) => { - // TODO - Truthiness::Ambiguous - } - - Type::LiteralValue(literal) => match literal.kind() { - LiteralValueTypeKind::LiteralString => Truthiness::Ambiguous, - LiteralValueTypeKind::Enum(enum_type) => enum_type - .enum_class_instance(db) - .try_bool_impl(db, allow_short_circuit, visitor)?, - - LiteralValueTypeKind::Int(num) => Truthiness::from(num.as_i64() != 0), - LiteralValueTypeKind::Bool(bool) => Truthiness::from(bool), - LiteralValueTypeKind::String(str) => Truthiness::from(!str.value(db).is_empty()), - LiteralValueTypeKind::Bytes(bytes) => Truthiness::from(!bytes.value(db).is_empty()), - }, - - Type::TypeAlias(alias) => visitor.visit(*self, || { - alias - .value_type(db) - .try_bool_impl(db, allow_short_circuit, visitor) - })?, - Type::NewTypeInstance(newtype) => { - newtype - .concrete_base_type(db) - .try_bool_impl(db, allow_short_circuit, visitor)? - } - }; - - Ok(truthiness) - } - /// Return the type of `len()` on a type if it is known more precisely than `int`, /// or `None` otherwise. /// @@ -5243,433 +4934,6 @@ impl<'db> Type<'db> { } } - /// Returns a tuple spec describing the elements that are produced when iterating over `self`. - /// - /// This method should only be used outside of type checking because it omits any errors. - /// For type checking, use [`try_iterate`](Self::try_iterate) instead. - fn iterate(self, db: &'db dyn Db) -> Cow<'db, TupleSpec<'db>> { - self.try_iterate(db) - .unwrap_or_else(|err| Cow::Owned(TupleSpec::homogeneous(err.fallback_element_type(db)))) - } - - /// Given the type of an object that is iterated over in some way, - /// return a tuple spec describing the type of objects that are yielded by that iteration. - /// - /// E.g., for the following call, given the type of `x`, infer the types of the values that are - /// splatted into `y`'s positional arguments: - /// ```python - /// y(*x) - /// ``` - fn try_iterate(self, db: &'db dyn Db) -> Result>, IterationError<'db>> { - self.try_iterate_with_mode(db, EvaluationMode::Sync) - } - - fn try_iterate_with_mode( - self, - db: &'db dyn Db, - mode: EvaluationMode, - ) -> Result>, IterationError<'db>> { - fn non_async_special_case<'db>( - db: &'db dyn Db, - ty: Type<'db>, - ) -> Option>> { - // We will not infer precise heterogeneous tuple specs for literals with lengths above this threshold. - // The threshold here is somewhat arbitrary and conservative; it could be increased if needed. - // However, it's probably very rare to need heterogeneous unpacking inference for long string literals - // or bytes literals, and creating long heterogeneous tuple specs has a performance cost. - const MAX_TUPLE_LENGTH: usize = 128; - - match ty { - Type::NominalInstance(nominal) => nominal.tuple_spec(db), - Type::NewTypeInstance(newtype) => non_async_special_case(db, newtype.concrete_base_type(db)), - Type::GenericAlias(alias) if alias.origin(db).is_tuple(db) => { - Some(Cow::Owned(TupleSpec::homogeneous(todo_type!( - "*tuple[] annotations" - )))) - } - Type::LiteralValue(literal) => match literal.kind() { - LiteralValueTypeKind::Bytes(bytes) => { - let bytes_literal = bytes.value(db); - let spec = if bytes_literal.len() < MAX_TUPLE_LENGTH { - TupleSpec::heterogeneous( - bytes_literal - .iter() - .map(|b| Type::int_literal( i64::from(*b))), - ) - } else { - TupleSpec::homogeneous(KnownClass::Int.to_instance(db)) - }; - Some(Cow::Owned(spec)) - }, - LiteralValueTypeKind::String(string_literal_ty) => { - let string_literal = string_literal_ty.value(db); - let spec = if string_literal.len() < MAX_TUPLE_LENGTH { - TupleSpec::heterogeneous( - string_literal - .chars() - .map(|c| Type::string_literal(db, &c.to_string())), - ) - } else { - TupleSpec::homogeneous(Type::literal_string()) - }; - Some(Cow::Owned(spec)) - } - // N.B. This special case isn't strictly necessary, it's just an obvious optimization - LiteralValueTypeKind::LiteralString => { - Some(Cow::Owned(TupleSpec::homogeneous(ty))) - } - _ => None - } - Type::Never => { - // The dunder logic below would have us return `tuple[Never, ...]`, which eagerly - // simplifies to `tuple[()]`. That will will cause us to emit false positives if we - // index into the tuple. Using `tuple[Unknown, ...]` avoids these false positives. - // TODO: Consider removing this special case, and instead hide the indexing - // diagnostic in unreachable code. - Some(Cow::Owned(TupleSpec::homogeneous(Type::unknown()))) - } - Type::TypeAlias(alias) => { - non_async_special_case(db, alias.value_type(db)) - } - Type::TypeVar(tvar) => match tvar.typevar(db).bound_or_constraints(db)? { - TypeVarBoundOrConstraints::UpperBound(bound) => { - non_async_special_case(db, bound) - } - TypeVarBoundOrConstraints::Constraints(constraints) => non_async_special_case(db, constraints.as_type(db)), - }, - Type::Union(union) => { - let elements = union.elements(db); - if elements.len() < MAX_TUPLE_LENGTH { - let mut elements_iter = elements.iter(); - let first_element_spec = elements_iter.next()?.try_iterate_with_mode(db, EvaluationMode::Sync).ok()?; - let mut builder = TupleSpecBuilder::from(&*first_element_spec); - for element in elements_iter { - builder = builder.union(db, &*element.try_iterate_with_mode(db, EvaluationMode::Sync).ok()?); - } - Some(Cow::Owned(builder.build())) - } else { - None - } - } - Type::Intersection(intersection) => { - // For intersections containing TypeVars with union bounds, we need to - // flatten the TypeVars first. This distributes the intersection over - // the union and simplifies, e.g.: - // `T & tuple[object, ...]` where `T: tuple[int, ...] | list[str]` - // becomes `(tuple[int, ...] & tuple[object, ...]) | (list[str] & tuple[object, ...])` - // which simplifies to `tuple[int, ...] | Never` = `tuple[int, ...]` - // - // After flattening, the result may be: - // - An intersection (if no union-bound typevars, or they didn't simplify). - // - A union of intersections (if distribution happened). - // - A simpler type (if it fully simplified). - // - // We then iterate over the flattened type. - let flattened = ty.flatten_typevars(db); - - // If flattening didn't change anything, iterate the intersection directly. - if flattened == ty { - let mut specs_iter = intersection.positive_elements_or_object(db).filter_map( - |element| element.try_iterate_with_mode(db, EvaluationMode::Sync).ok(), - ); - let first_spec = specs_iter.next()?; - let mut builder = TupleSpecBuilder::from(&*first_spec); - for spec in specs_iter { - // Two tuples cannot have incompatible specs unless the tuples themselves - // are disjoint. `IntersectionBuilder` eagerly simplifies such - // intersections to `Never`, so this should always return `Some`. - let Some(intersected) = builder.intersect(db, &spec) else { - return Some(Cow::Owned(TupleSpec::homogeneous(Type::unknown()))); - }; - builder = intersected; - } - return Some(Cow::Owned(builder.build())); - } - - // Flattening changed the type; recursively iterate the flattened result. - non_async_special_case(db, flattened) - } - // N.B. This special case isn't strictly necessary, it's just an obvious optimization - Type::Dynamic(_) => Some(Cow::Owned(TupleSpec::homogeneous(ty))), - - Type::FunctionLiteral(_) - | Type::GenericAlias(_) - | Type::BoundMethod(_) - | Type::KnownBoundMethod(_) - | Type::WrapperDescriptor(_) - | Type::DataclassDecorator(_) - | Type::DataclassTransformer(_) - | Type::Callable(_) - | Type::ModuleLiteral(_) - // We could infer a precise tuple spec for enum classes with members, - // but it's not clear whether that's worth the added complexity: - // you'd have to check that `EnumMeta.__iter__` is not overridden for it to be sound - // (enums can have `EnumMeta` subclasses as their metaclasses). - | Type::ClassLiteral(_) - | Type::SubclassOf(_) - | Type::ProtocolInstance(_) - | Type::SpecialForm(_) - | Type::KnownInstance(_) - | Type::PropertyInstance(_) - | Type::AlwaysTruthy - | Type::AlwaysFalsy - | Type::BoundSuper(_) - | Type::TypeIs(_) - | Type::TypeGuard(_) - | Type::TypedDict(_) => None - } - } - - if mode.is_async() { - let try_call_dunder_anext_on_iterator = |iterator: Type<'db>| -> Result< - Result, AwaitError<'db>>, - CallDunderError<'db>, - > { - iterator - .try_call_dunder( - db, - "__anext__", - CallArguments::none(), - TypeContext::default(), - ) - .map(|dunder_anext_outcome| dunder_anext_outcome.return_type(db).try_await(db)) - }; - - return match self.try_call_dunder( - db, - "__aiter__", - CallArguments::none(), - TypeContext::default(), - ) { - Ok(dunder_aiter_bindings) => { - let iterator = dunder_aiter_bindings.return_type(db); - match try_call_dunder_anext_on_iterator(iterator) { - Ok(Ok(result)) => Ok(Cow::Owned(TupleSpec::homogeneous(result))), - Ok(Err(AwaitError::InvalidReturnType(..))) => { - Err(IterationError::UnboundAiterError) - } // TODO: __anext__ is bound, but is not properly awaitable - Err(dunder_anext_error) | Ok(Err(AwaitError::Call(dunder_anext_error))) => { - Err(IterationError::IterReturnsInvalidIterator { - iterator, - dunder_error: dunder_anext_error, - mode, - }) - } - } - } - Err(CallDunderError::PossiblyUnbound(dunder_aiter_bindings)) => { - let iterator = dunder_aiter_bindings.return_type(db); - match try_call_dunder_anext_on_iterator(iterator) { - Ok(_) => Err(IterationError::IterCallError { - kind: CallErrorKind::PossiblyNotCallable, - bindings: dunder_aiter_bindings, - mode, - }), - Err(dunder_anext_error) => { - Err(IterationError::IterReturnsInvalidIterator { - iterator, - dunder_error: dunder_anext_error, - mode, - }) - } - } - } - Err(CallDunderError::CallError(kind, bindings)) => { - Err(IterationError::IterCallError { - kind, - bindings, - mode, - }) - } - Err(CallDunderError::MethodNotAvailable) => Err(IterationError::UnboundAiterError), - }; - } - - if let Some(special_case) = non_async_special_case(db, self) { - return Ok(special_case); - } - - let try_call_dunder_getitem = || { - self.try_call_dunder( - db, - "__getitem__", - CallArguments::positional([KnownClass::Int.to_instance(db)]), - TypeContext::default(), - ) - .map(|dunder_getitem_outcome| dunder_getitem_outcome.return_type(db)) - }; - - let try_call_dunder_next_on_iterator = |iterator: Type<'db>| { - iterator - .try_call_dunder( - db, - "__next__", - CallArguments::none(), - TypeContext::default(), - ) - .map(|dunder_next_outcome| dunder_next_outcome.return_type(db)) - }; - - let dunder_iter_result = self - .try_call_dunder( - db, - "__iter__", - CallArguments::none(), - TypeContext::default(), - ) - .map(|dunder_iter_outcome| dunder_iter_outcome.return_type(db)); - - match dunder_iter_result { - Ok(iterator) => { - // `__iter__` is definitely bound and calling it succeeds. - // See what calling `__next__` on the object returned by `__iter__` gives us... - try_call_dunder_next_on_iterator(iterator) - .map(|ty| Cow::Owned(TupleSpec::homogeneous(ty))) - .map_err( - |dunder_next_error| IterationError::IterReturnsInvalidIterator { - iterator, - dunder_error: dunder_next_error, - mode, - }, - ) - } - - // `__iter__` is possibly unbound... - Err(CallDunderError::PossiblyUnbound(dunder_iter_outcome)) => { - let iterator = dunder_iter_outcome.return_type(db); - - match try_call_dunder_next_on_iterator(iterator) { - Ok(dunder_next_return) => { - try_call_dunder_getitem() - .map(|dunder_getitem_return_type| { - // If `__iter__` is possibly unbound, - // but it returns an object that has a bound and valid `__next__` method, - // *and* the object has a bound and valid `__getitem__` method, - // we infer a union of the type returned by the `__next__` method - // and the type returned by the `__getitem__` method. - // - // No diagnostic is emitted; iteration will always succeed! - Cow::Owned(TupleSpec::homogeneous(UnionType::from_two_elements( - db, - dunder_next_return, - dunder_getitem_return_type, - ))) - }) - .map_err(|dunder_getitem_error| { - IterationError::PossiblyUnboundIterAndGetitemError { - dunder_next_return, - dunder_getitem_error, - } - }) - } - - Err(dunder_next_error) => Err(IterationError::IterReturnsInvalidIterator { - iterator, - dunder_error: dunder_next_error, - mode, - }), - } - } - - // `__iter__` is definitely bound but it can't be called with the expected arguments - Err(CallDunderError::CallError(kind, bindings)) => Err(IterationError::IterCallError { - kind, - bindings, - mode, - }), - - // There's no `__iter__` method. Try `__getitem__` instead... - Err(CallDunderError::MethodNotAvailable) => try_call_dunder_getitem() - .map(|ty| Cow::Owned(TupleSpec::homogeneous(ty))) - .map_err( - |dunder_getitem_error| IterationError::UnboundIterAndGetitemError { - dunder_getitem_error, - }, - ), - } - } - - /// Returns the type bound from a context manager with type `self`. - /// - /// This method should only be used outside of type checking because it omits any errors. - /// For type checking, use [`try_enter_with_mode`](Self::try_enter_with_mode) instead. - fn enter(self, db: &'db dyn Db) -> Type<'db> { - self.try_enter_with_mode(db, EvaluationMode::Sync) - .unwrap_or_else(|err| err.fallback_enter_type(db)) - } - - /// Returns the type bound from a context manager with type `self`. - /// - /// This method should only be used outside of type checking because it omits any errors. - /// For type checking, use [`try_enter_with_mode`](Self::try_enter_with_mode) instead. - fn aenter(self, db: &'db dyn Db) -> Type<'db> { - self.try_enter_with_mode(db, EvaluationMode::Async) - .unwrap_or_else(|err| err.fallback_enter_type(db)) - } - - /// Given the type of an object that is used as a context manager (i.e. in a `with` statement), - /// return the return type of its `__enter__` or `__aenter__` method, which is bound to any potential targets. - /// - /// E.g., for the following `with` statement, given the type of `x`, infer the type of `y`: - /// ```python - /// with x as y: - /// pass - /// ``` - fn try_enter_with_mode( - self, - db: &'db dyn Db, - mode: EvaluationMode, - ) -> Result, ContextManagerError<'db>> { - let (enter_method, exit_method) = match mode { - EvaluationMode::Async => ("__aenter__", "__aexit__"), - EvaluationMode::Sync => ("__enter__", "__exit__"), - }; - - let enter = self.try_call_dunder( - db, - enter_method, - CallArguments::none(), - TypeContext::default(), - ); - let exit = self.try_call_dunder( - db, - exit_method, - CallArguments::positional([Type::none(db), Type::none(db), Type::none(db)]), - TypeContext::default(), - ); - - // TODO: Make use of Protocols when we support it (the manager be assignable to `contextlib.AbstractContextManager`). - match (enter, exit) { - (Ok(enter), Ok(_)) => { - let ty = enter.return_type(db); - Ok(if mode.is_async() { - ty.try_await(db).unwrap_or(Type::unknown()) - } else { - ty - }) - } - (Ok(enter), Err(exit_error)) => { - let ty = enter.return_type(db); - Err(ContextManagerError::Exit { - enter_return_type: if mode.is_async() { - ty.try_await(db).unwrap_or(Type::unknown()) - } else { - ty - }, - exit_error, - mode, - }) - } - // TODO: Use the `exit_ty` to determine if any raised exception is suppressed. - (Err(enter_error), Ok(_)) => Err(ContextManagerError::Enter(enter_error, mode)), - (Err(enter_error), Err(exit_error)) => Err(ContextManagerError::EnterAndExit { - enter_error, - exit_error, - mode, - }), - } - } - /// Resolve the type of an `await …` expression where `self` is the type of the awaitable. fn try_await(self, db: &'db dyn Db) -> Result, AwaitError<'db>> { let await_result = self.try_call_dunder( @@ -8920,791 +8184,6 @@ impl<'db> AwaitError<'db> { } } -/// Error returned if a type is not (or may not be) a context manager. -#[derive(Debug)] -enum ContextManagerError<'db> { - Enter(CallDunderError<'db>, EvaluationMode), - Exit { - enter_return_type: Type<'db>, - exit_error: CallDunderError<'db>, - mode: EvaluationMode, - }, - EnterAndExit { - enter_error: CallDunderError<'db>, - exit_error: CallDunderError<'db>, - mode: EvaluationMode, - }, -} - -impl<'db> ContextManagerError<'db> { - fn fallback_enter_type(&self, db: &'db dyn Db) -> Type<'db> { - self.enter_type(db).unwrap_or(Type::unknown()) - } - - /// Returns the `__enter__` or `__aenter__` return type if it is known, - /// or `None` if the type never has a callable `__enter__` or `__aenter__` attribute - fn enter_type(&self, db: &'db dyn Db) -> Option> { - match self { - Self::Exit { - enter_return_type, - exit_error: _, - mode: _, - } => Some(*enter_return_type), - Self::Enter(enter_error, _) - | Self::EnterAndExit { - enter_error, - exit_error: _, - mode: _, - } => match enter_error { - CallDunderError::PossiblyUnbound(call_outcome) => { - Some(call_outcome.return_type(db)) - } - CallDunderError::CallError(CallErrorKind::NotCallable, _) => None, - CallDunderError::CallError(_, bindings) => Some(bindings.return_type(db)), - CallDunderError::MethodNotAvailable => None, - }, - } - } - - fn report_diagnostic( - &self, - context: &InferContext<'db, '_>, - context_expression_type: Type<'db>, - context_expression_node: ast::AnyNodeRef, - ) { - let Some(builder) = context.report_lint(&INVALID_CONTEXT_MANAGER, context_expression_node) - else { - return; - }; - - let mode = match self { - Self::Exit { mode, .. } | Self::Enter(_, mode) | Self::EnterAndExit { mode, .. } => { - *mode - } - }; - - let (enter_method, exit_method) = match mode { - EvaluationMode::Async => ("__aenter__", "__aexit__"), - EvaluationMode::Sync => ("__enter__", "__exit__"), - }; - - let format_call_dunder_error = |call_dunder_error: &CallDunderError<'db>, name: &str| { - match call_dunder_error { - CallDunderError::MethodNotAvailable => format!("it does not implement `{name}`"), - CallDunderError::PossiblyUnbound(_) => { - format!("the method `{name}` may be missing") - } - // TODO: Use more specific error messages for the different error cases. - // E.g. hint toward the union variant that doesn't correctly implement enter, - // distinguish between a not callable `__enter__` attribute and a wrong signature. - CallDunderError::CallError(_, _) => { - format!("it does not correctly implement `{name}`") - } - } - }; - - let format_call_dunder_errors = |error_a: &CallDunderError<'db>, - name_a: &str, - error_b: &CallDunderError<'db>, - name_b: &str| { - match (error_a, error_b) { - (CallDunderError::PossiblyUnbound(_), CallDunderError::PossiblyUnbound(_)) => { - format!("the methods `{name_a}` and `{name_b}` are possibly missing") - } - (CallDunderError::MethodNotAvailable, CallDunderError::MethodNotAvailable) => { - format!("it does not implement `{name_a}` and `{name_b}`") - } - (CallDunderError::CallError(_, _), CallDunderError::CallError(_, _)) => { - format!("it does not correctly implement `{name_a}` or `{name_b}`") - } - (_, _) => format!( - "{format_a}, and {format_b}", - format_a = format_call_dunder_error(error_a, name_a), - format_b = format_call_dunder_error(error_b, name_b) - ), - } - }; - - let db = context.db(); - - let formatted_errors = match self { - Self::Exit { - enter_return_type: _, - exit_error, - mode: _, - } => format_call_dunder_error(exit_error, exit_method), - Self::Enter(enter_error, _) => format_call_dunder_error(enter_error, enter_method), - Self::EnterAndExit { - enter_error, - exit_error, - mode: _, - } => format_call_dunder_errors(enter_error, enter_method, exit_error, exit_method), - }; - - // Suggest using `async with` if only async methods are available in a sync context, - // or suggest using `with` if only sync methods are available in an async context. - let with_kw = match mode { - EvaluationMode::Sync => "with", - EvaluationMode::Async => "async with", - }; - - let mut diag = builder.into_diagnostic(format_args!( - "Object of type `{}` cannot be used with `{}` because {}", - context_expression_type.display(db), - with_kw, - formatted_errors, - )); - - let (alt_mode, alt_enter_method, alt_exit_method, alt_with_kw) = match mode { - EvaluationMode::Sync => ("async", "__aenter__", "__aexit__", "async with"), - EvaluationMode::Async => ("sync", "__enter__", "__exit__", "with"), - }; - - let alt_enter = context_expression_type.try_call_dunder( - db, - alt_enter_method, - CallArguments::none(), - TypeContext::default(), - ); - let alt_exit = context_expression_type.try_call_dunder( - db, - alt_exit_method, - CallArguments::positional([Type::unknown(), Type::unknown(), Type::unknown()]), - TypeContext::default(), - ); - - if (alt_enter.is_ok() || matches!(alt_enter, Err(CallDunderError::CallError(..)))) - && (alt_exit.is_ok() || matches!(alt_exit, Err(CallDunderError::CallError(..)))) - { - diag.info(format_args!( - "Objects of type `{}` can be used as {} context managers", - context_expression_type.display(db), - alt_mode - )); - diag.info(format!("Consider using `{alt_with_kw}` here")); - } - } -} - -/// Error returned if a type is not (or may not be) iterable. -#[derive(Debug)] -enum IterationError<'db> { - /// The object being iterated over has a bound `__(a)iter__` method, - /// but calling it with the expected arguments results in an error. - IterCallError { - kind: CallErrorKind, - bindings: Box>, - mode: EvaluationMode, - }, - - /// The object being iterated over has a bound `__(a)iter__` method that can be called - /// with the expected types, but it returns an object that is not a valid iterator. - IterReturnsInvalidIterator { - /// The type of the object returned by the `__(a)iter__` method. - iterator: Type<'db>, - /// The error we encountered when we tried to call `__(a)next__` on the type - /// returned by `__(a)iter__` - dunder_error: CallDunderError<'db>, - /// Whether this is a synchronous or an asynchronous iterator. - mode: EvaluationMode, - }, - - /// The object being iterated over has a bound `__iter__` method that returns a - /// valid iterator. However, the `__iter__` method is possibly unbound, and there - /// either isn't a `__getitem__` method to fall back to, or calling the `__getitem__` - /// method returns some kind of error. - PossiblyUnboundIterAndGetitemError { - /// The type of the object returned by the `__next__` method on the iterator. - /// (The iterator being the type returned by the `__iter__` method on the iterable.) - dunder_next_return: Type<'db>, - /// The error we encountered when we tried to call `__getitem__` on the iterable. - dunder_getitem_error: CallDunderError<'db>, - }, - - /// The object being iterated over doesn't have an `__iter__` method. - /// It also either doesn't have a `__getitem__` method to fall back to, - /// or calling the `__getitem__` method returns some kind of error. - UnboundIterAndGetitemError { - dunder_getitem_error: CallDunderError<'db>, - }, - - /// The asynchronous iterable has no `__aiter__` method. - UnboundAiterError, -} - -impl<'db> IterationError<'db> { - fn fallback_element_type(&self, db: &'db dyn Db) -> Type<'db> { - self.element_type(db).unwrap_or(Type::unknown()) - } - - /// Returns the element type if it is known, or `None` if the type is never iterable. - fn element_type(&self, db: &'db dyn Db) -> Option> { - let return_type = |result: Result, CallDunderError<'db>>| { - result - .map(|outcome| Some(outcome.return_type(db))) - .unwrap_or_else(|call_error| call_error.return_type(db)) - }; - - match self { - Self::IterReturnsInvalidIterator { - dunder_error, mode, .. - } => dunder_error.return_type(db).and_then(|ty| { - if mode.is_async() { - ty.try_await(db).ok() - } else { - Some(ty) - } - }), - - Self::IterCallError { - kind: _, - bindings: dunder_iter_bindings, - mode, - } => { - if mode.is_async() { - return_type(dunder_iter_bindings.return_type(db).try_call_dunder( - db, - "__anext__", - CallArguments::none(), - TypeContext::default(), - )) - .and_then(|ty| ty.try_await(db).ok()) - } else { - return_type(dunder_iter_bindings.return_type(db).try_call_dunder( - db, - "__next__", - CallArguments::none(), - TypeContext::default(), - )) - } - } - - Self::PossiblyUnboundIterAndGetitemError { - dunder_next_return, - dunder_getitem_error, - } => match dunder_getitem_error { - CallDunderError::MethodNotAvailable => Some(*dunder_next_return), - CallDunderError::PossiblyUnbound(dunder_getitem_outcome) => { - Some(UnionType::from_two_elements( - db, - *dunder_next_return, - dunder_getitem_outcome.return_type(db), - )) - } - CallDunderError::CallError(CallErrorKind::NotCallable, _) => { - Some(*dunder_next_return) - } - CallDunderError::CallError(_, dunder_getitem_bindings) => { - let dunder_getitem_return = dunder_getitem_bindings.return_type(db); - Some(UnionType::from_two_elements( - db, - *dunder_next_return, - dunder_getitem_return, - )) - } - }, - - Self::UnboundIterAndGetitemError { - dunder_getitem_error, - } => dunder_getitem_error.return_type(db), - - Self::UnboundAiterError => None, - } - } - - /// Does this error concern a synchronous or asynchronous iterable? - fn mode(&self) -> EvaluationMode { - match self { - Self::IterCallError { mode, .. } => *mode, - Self::IterReturnsInvalidIterator { mode, .. } => *mode, - Self::PossiblyUnboundIterAndGetitemError { .. } - | Self::UnboundIterAndGetitemError { .. } => EvaluationMode::Sync, - Self::UnboundAiterError => EvaluationMode::Async, - } - } - - /// Reports the diagnostic for this error. - fn report_diagnostic( - &self, - context: &InferContext<'db, '_>, - iterable_type: Type<'db>, - iterable_node: ast::AnyNodeRef, - ) { - /// A little helper type for emitting a diagnostic - /// based on the variant of iteration error. - struct Reporter<'a> { - db: &'a dyn Db, - builder: LintDiagnosticGuardBuilder<'a, 'a>, - iterable_type: Type<'a>, - mode: EvaluationMode, - } - - impl<'a> Reporter<'a> { - /// Emit a diagnostic that is certain that `iterable_type` is not iterable. - /// - /// `because` should explain why `iterable_type` is not iterable. - #[expect(clippy::wrong_self_convention)] - fn is_not(self, because: impl std::fmt::Display) -> LintDiagnosticGuard<'a, 'a> { - let mut diag = self.builder.into_diagnostic(format_args!( - "Object of type `{iterable_type}` is not {maybe_async}iterable", - iterable_type = self.iterable_type.display(self.db), - maybe_async = if self.mode.is_async() { "async-" } else { "" } - )); - diag.info(because); - diag - } - - /// Emit a diagnostic that is uncertain that `iterable_type` is not iterable. - /// - /// `because` should explain why `iterable_type` is likely not iterable. - fn may_not(self, because: impl std::fmt::Display) -> LintDiagnosticGuard<'a, 'a> { - let mut diag = self.builder.into_diagnostic(format_args!( - "Object of type `{iterable_type}` may not be {maybe_async}iterable", - iterable_type = self.iterable_type.display(self.db), - maybe_async = if self.mode.is_async() { "async-" } else { "" } - )); - diag.info(because); - diag - } - } - - let Some(builder) = context.report_lint(&NOT_ITERABLE, iterable_node) else { - return; - }; - let db = context.db(); - let mode = self.mode(); - let reporter = Reporter { - db, - builder, - iterable_type, - mode, - }; - - // TODO: for all of these error variants, the "explanation" for the diagnostic - // (everything after the "because") should really be presented as a "help:", "note", - // or similar, rather than as part of the same sentence as the error message. - match self { - Self::IterCallError { - kind, - bindings, - mode, - } => { - let method = if mode.is_async() { - "__aiter__" - } else { - "__iter__" - }; - - match kind { - CallErrorKind::NotCallable => { - reporter.is_not(format_args!( - "Its `{method}` attribute has type `{dunder_iter_type}`, which is not callable", - dunder_iter_type = bindings.callable_type().display(db), - )); - } - CallErrorKind::PossiblyNotCallable => { - reporter.may_not(format_args!( - "Its `{method}` attribute (with type `{dunder_iter_type}`) \ - may not be callable", - dunder_iter_type = bindings.callable_type().display(db), - )); - } - CallErrorKind::BindingError => { - if bindings.is_single() { - reporter - .is_not(format_args!( - "Its `{method}` method has an invalid signature" - )) - .info(format_args!("Expected signature `def {method}(self): ...`")); - } else { - let mut diag = reporter.may_not(format_args!( - "Its `{method}` method may have an invalid signature" - )); - diag.info(format_args!( - "Type of `{method}` is `{dunder_iter_type}`", - dunder_iter_type = bindings.callable_type().display(db), - )); - diag.info(format_args!( - "Expected signature for `{method}` is `def {method}(self): ...`", - )); - } - } - } - } - - Self::IterReturnsInvalidIterator { - iterator, - dunder_error: dunder_next_error, - mode, - } => { - let dunder_iter_name = if mode.is_async() { - "__aiter__" - } else { - "__iter__" - }; - let dunder_next_name = if mode.is_async() { - "__anext__" - } else { - "__next__" - }; - match dunder_next_error { - CallDunderError::MethodNotAvailable => { - reporter.is_not(format_args!( - "Its `{dunder_iter_name}` method returns an object of type `{iterator_type}`, \ - which has no `{dunder_next_name}` method", - iterator_type = iterator.display(db), - )); - } - CallDunderError::PossiblyUnbound(_) => { - reporter.may_not(format_args!( - "Its `{dunder_iter_name}` method returns an object of type `{iterator_type}`, \ - which may not have a `{dunder_next_name}` method", - iterator_type = iterator.display(db), - )); - } - CallDunderError::CallError(CallErrorKind::NotCallable, _) => { - reporter.is_not(format_args!( - "Its `{dunder_iter_name}` method returns an object of type `{iterator_type}`, \ - which has a `{dunder_next_name}` attribute that is not callable", - iterator_type = iterator.display(db), - )); - } - CallDunderError::CallError(CallErrorKind::PossiblyNotCallable, _) => { - reporter.may_not(format_args!( - "Its `{dunder_iter_name}` method returns an object of type `{iterator_type}`, \ - which has a `{dunder_next_name}` attribute that may not be callable", - iterator_type = iterator.display(db), - )); - } - CallDunderError::CallError(CallErrorKind::BindingError, bindings) - if bindings.is_single() => - { - reporter - .is_not(format_args!( - "Its `{dunder_iter_name}` method returns an object of type `{iterator_type}`, \ - which has an invalid `{dunder_next_name}` method", - iterator_type = iterator.display(db), - )) - .info(format_args!("Expected signature for `{dunder_next_name}` is `def {dunder_next_name}(self): ...`")); - } - CallDunderError::CallError(CallErrorKind::BindingError, _) => { - reporter - .may_not(format_args!( - "Its `{dunder_iter_name}` method returns an object of type `{iterator_type}`, \ - which may have an invalid `{dunder_next_name}` method", - iterator_type = iterator.display(db), - )) - .info(format_args!("Expected signature for `{dunder_next_name}` is `def {dunder_next_name}(self): ...`")); - } - } - } - - Self::PossiblyUnboundIterAndGetitemError { - dunder_getitem_error, - .. - } => match dunder_getitem_error { - CallDunderError::MethodNotAvailable => { - reporter.may_not( - "It may not have an `__iter__` method \ - and it doesn't have a `__getitem__` method", - ); - } - CallDunderError::PossiblyUnbound(_) => { - reporter - .may_not("It may not have an `__iter__` method or a `__getitem__` method"); - } - CallDunderError::CallError(CallErrorKind::NotCallable, bindings) => { - reporter.may_not(format_args!( - "It may not have an `__iter__` method \ - and its `__getitem__` attribute has type `{dunder_getitem_type}`, \ - which is not callable", - dunder_getitem_type = bindings.callable_type().display(db), - )); - } - CallDunderError::CallError(CallErrorKind::PossiblyNotCallable, bindings) - if bindings.is_single() => - { - reporter.may_not( - "It may not have an `__iter__` method \ - and its `__getitem__` attribute may not be callable", - ); - } - CallDunderError::CallError(CallErrorKind::PossiblyNotCallable, bindings) => { - reporter.may_not(format_args!( - "It may not have an `__iter__` method \ - and its `__getitem__` attribute (with type `{dunder_getitem_type}`) \ - may not be callable", - dunder_getitem_type = bindings.callable_type().display(db), - )); - } - CallDunderError::CallError(CallErrorKind::BindingError, bindings) - if bindings.is_single() => - { - reporter - .may_not( - "It may not have an `__iter__` method \ - and its `__getitem__` method has an incorrect signature \ - for the old-style iteration protocol", - ) - .info( - "`__getitem__` must be at least as permissive as \ - `def __getitem__(self, key: int): ...` \ - to satisfy the old-style iteration protocol", - ); - } - CallDunderError::CallError(CallErrorKind::BindingError, bindings) => { - reporter - .may_not(format_args!( - "It may not have an `__iter__` method \ - and its `__getitem__` method (with type `{dunder_getitem_type}`) \ - may have an incorrect signature for the old-style iteration protocol", - dunder_getitem_type = bindings.callable_type().display(db), - )) - .info( - "`__getitem__` must be at least as permissive as \ - `def __getitem__(self, key: int): ...` \ - to satisfy the old-style iteration protocol", - ); - } - }, - - Self::UnboundIterAndGetitemError { - dunder_getitem_error, - } => match dunder_getitem_error { - CallDunderError::MethodNotAvailable => { - reporter - .is_not("It doesn't have an `__iter__` method or a `__getitem__` method"); - } - CallDunderError::PossiblyUnbound(_) => { - reporter.is_not( - "It has no `__iter__` method and it may not have a `__getitem__` method", - ); - } - CallDunderError::CallError(CallErrorKind::NotCallable, bindings) => { - reporter.is_not(format_args!( - "It has no `__iter__` method and \ - its `__getitem__` attribute has type `{dunder_getitem_type}`, \ - which is not callable", - dunder_getitem_type = bindings.callable_type().display(db), - )); - } - CallDunderError::CallError(CallErrorKind::PossiblyNotCallable, bindings) - if bindings.is_single() => - { - reporter.may_not( - "It has no `__iter__` method and its `__getitem__` attribute \ - may not be callable", - ); - } - CallDunderError::CallError(CallErrorKind::PossiblyNotCallable, bindings) => { - reporter.may_not( - "It has no `__iter__` method and its `__getitem__` attribute is invalid", - ).info(format_args!( - "`__getitem__` has type `{dunder_getitem_type}`, which is not callable", - dunder_getitem_type = bindings.callable_type().display(db), - )); - } - CallDunderError::CallError(CallErrorKind::BindingError, bindings) - if bindings.is_single() => - { - reporter - .is_not( - "It has no `__iter__` method and \ - its `__getitem__` method has an incorrect signature \ - for the old-style iteration protocol", - ) - .info( - "`__getitem__` must be at least as permissive as \ - `def __getitem__(self, key: int): ...` \ - to satisfy the old-style iteration protocol", - ); - } - CallDunderError::CallError(CallErrorKind::BindingError, bindings) => { - reporter - .may_not(format_args!( - "It has no `__iter__` method and \ - its `__getitem__` method (with type `{dunder_getitem_type}`) \ - may have an incorrect signature for the old-style iteration protocol", - dunder_getitem_type = bindings.callable_type().display(db), - )) - .info( - "`__getitem__` must be at least as permissive as \ - `def __getitem__(self, key: int): ...` \ - to satisfy the old-style iteration protocol", - ); - } - }, - - IterationError::UnboundAiterError => { - reporter.is_not("It has no `__aiter__` method"); - } - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(super) enum BoolError<'db> { - /// The type has a `__bool__` attribute but it can't be called. - NotCallable { not_boolable_type: Type<'db> }, - - /// The type has a callable `__bool__` attribute, but it isn't callable - /// with the given arguments. - IncorrectArguments { - not_boolable_type: Type<'db>, - truthiness: Truthiness, - }, - - /// The type has a `__bool__` method, is callable with the given arguments, - /// but the return type isn't assignable to `bool`. - IncorrectReturnType { - not_boolable_type: Type<'db>, - return_type: Type<'db>, - }, - - /// A union type doesn't implement `__bool__` correctly. - Union { - union: UnionType<'db>, - truthiness: Truthiness, - }, - - /// Any other reason why the type can't be converted to a bool. - /// E.g. because calling `__bool__` returns in a union type and not all variants support `__bool__` or - /// because `__bool__` points to a type that has a possibly missing `__call__` method. - Other { not_boolable_type: Type<'db> }, -} - -impl<'db> BoolError<'db> { - pub(super) fn fallback_truthiness(&self) -> Truthiness { - match self { - BoolError::NotCallable { .. } - | BoolError::IncorrectReturnType { .. } - | BoolError::Other { .. } => Truthiness::Ambiguous, - BoolError::IncorrectArguments { truthiness, .. } - | BoolError::Union { truthiness, .. } => *truthiness, - } - } - - fn not_boolable_type(&self) -> Type<'db> { - match self { - BoolError::NotCallable { - not_boolable_type, .. - } - | BoolError::IncorrectArguments { - not_boolable_type, .. - } - | BoolError::Other { not_boolable_type } - | BoolError::IncorrectReturnType { - not_boolable_type, .. - } => *not_boolable_type, - BoolError::Union { union, .. } => Type::Union(*union), - } - } - - pub(super) fn report_diagnostic(&self, context: &InferContext, condition: impl Ranged) { - self.report_diagnostic_impl(context, condition.range()); - } - - fn report_diagnostic_impl(&self, context: &InferContext, condition: TextRange) { - let Some(builder) = context.report_lint(&UNSUPPORTED_BOOL_CONVERSION, condition) else { - return; - }; - match self { - Self::IncorrectArguments { - not_boolable_type, .. - } => { - let mut diag = builder.into_diagnostic(format_args!( - "Boolean conversion is not supported for type `{}`", - not_boolable_type.display(context.db()) - )); - let mut sub = SubDiagnostic::new( - SubDiagnosticSeverity::Info, - "`__bool__` methods must only have a `self` parameter", - ); - if let Some((func_span, parameter_span)) = not_boolable_type - .member(context.db(), "__bool__") - .into_lookup_result(context.db()) - .ok() - .and_then(|quals| quals.inner_type().parameter_span(context.db(), None)) - { - sub.annotate( - Annotation::primary(parameter_span).message("Incorrect parameters"), - ); - sub.annotate(Annotation::secondary(func_span).message("Method defined here")); - } - diag.sub(sub); - } - Self::IncorrectReturnType { - not_boolable_type, - return_type, - } => { - let mut diag = builder.into_diagnostic(format_args!( - "Boolean conversion is not supported for type `{not_boolable}`", - not_boolable = not_boolable_type.display(context.db()), - )); - let mut sub = SubDiagnostic::new( - SubDiagnosticSeverity::Info, - format_args!( - "`{return_type}` is not assignable to `bool`", - return_type = return_type.display(context.db()), - ), - ); - if let Some((func_span, return_type_span)) = not_boolable_type - .member(context.db(), "__bool__") - .into_lookup_result(context.db()) - .ok() - .and_then(|quals| quals.inner_type().function_spans(context.db())) - .and_then(|spans| Some((spans.name, spans.return_type?))) - { - sub.annotate( - Annotation::primary(return_type_span).message("Incorrect return type"), - ); - sub.annotate(Annotation::secondary(func_span).message("Method defined here")); - } - diag.sub(sub); - } - Self::NotCallable { not_boolable_type } => { - let mut diag = builder.into_diagnostic(format_args!( - "Boolean conversion is not supported for type `{}`", - not_boolable_type.display(context.db()) - )); - let sub = SubDiagnostic::new( - SubDiagnosticSeverity::Info, - format_args!( - "`__bool__` on `{}` must be callable", - not_boolable_type.display(context.db()) - ), - ); - // TODO: It would be nice to create an annotation here for - // where `__bool__` is defined. At time of writing, I couldn't - // figure out a straight-forward way of doing this. ---AG - diag.sub(sub); - } - Self::Union { union, .. } => { - let first_error = union - .elements(context.db()) - .iter() - .find_map(|element| element.try_bool(context.db()).err()) - .unwrap(); - - builder.into_diagnostic(format_args!( - "Boolean conversion is not supported for union `{}` \ - because `{}` doesn't implement `__bool__` correctly", - Type::Union(*union).display(context.db()), - first_error.not_boolable_type().display(context.db()), - )); - } - - Self::Other { not_boolable_type } => { - builder.into_diagnostic(format_args!( - "Boolean conversion is not supported for type `{}`; \ - it incorrectly implements `__bool__`", - not_boolable_type.display(context.db()) - )); - } - } - } -} - #[derive(Debug, Copy, Clone, PartialEq, Eq, get_size2::GetSize)] pub enum Truthiness { /// For an object `x`, `bool(x)` will always return `True` @@ -11391,6 +9870,26 @@ pub(super) fn determine_upper_bound<'db>( Type::instance(db, upper_bound) } +#[derive(Clone, Copy, Debug, Hash, salsa::Update, get_size2::GetSize)] +pub(crate) enum EvaluationMode { + Sync, + Async, +} + +impl EvaluationMode { + pub(crate) const fn from_is_async(is_async: bool) -> Self { + if is_async { + EvaluationMode::Async + } else { + EvaluationMode::Sync + } + } + + pub(crate) const fn is_async(self) -> bool { + matches!(self, EvaluationMode::Async) + } +} + // Make sure that the `Type` enum does not grow unexpectedly. #[cfg(not(debug_assertions))] #[cfg(target_pointer_width = "64")] diff --git a/crates/ty_python_semantic/src/types/bool.rs b/crates/ty_python_semantic/src/types/bool.rs new file mode 100644 index 0000000000000..da5291e781a5a --- /dev/null +++ b/crates/ty_python_semantic/src/types/bool.rs @@ -0,0 +1,487 @@ +use ruff_db::diagnostic::{Annotation, SubDiagnostic, SubDiagnosticSeverity}; +use ruff_text_size::{Ranged, TextRange}; + +use crate::{ + Db, + types::{ + CallArguments, CallDunderError, ClassType, CycleDetector, KnownClass, KnownInstanceType, + LiteralValueTypeKind, SubclassOfInner, Truthiness, Type, TypeContext, + TypeVarBoundOrConstraints, UnionType, call::CallErrorKind, + constraints::ConstraintSetBuilder, context::InferContext, + diagnostic::UNSUPPORTED_BOOL_CONVERSION, typed_dict::TypedDictField, + }, +}; + +impl<'db> Type<'db> { + /// Resolves the boolean value of the type and falls back to [`Truthiness::Ambiguous`] if the type doesn't implement `__bool__` correctly. + /// + /// This method should only be used outside type checking or when evaluating if a type + /// is truthy or falsy in a context where Python doesn't make an implicit `bool` call. + /// Use [`try_bool`](Self::try_bool) for type checking or implicit `bool` calls. + pub(crate) fn bool(&self, db: &'db dyn Db) -> Truthiness { + self.try_bool_impl(db, true, &TryBoolVisitor::new(Ok(Truthiness::Ambiguous))) + .unwrap_or_else(|err| err.fallback_truthiness()) + } + + /// Resolves the boolean value of a type. + /// + /// This is used to determine the value that would be returned + /// when `bool(x)` is called on an object `x`. + /// + /// Returns an error if the type doesn't implement `__bool__` correctly. + pub(crate) fn try_bool(&self, db: &'db dyn Db) -> Result> { + self.try_bool_impl(db, false, &TryBoolVisitor::new(Ok(Truthiness::Ambiguous))) + } + + /// Resolves the boolean value of a type. + /// + /// Setting `allow_short_circuit` to `true` allows the implementation to + /// early return if the bool value of any union variant is `Truthiness::Ambiguous`. + /// Early returning shows a 1-2% perf improvement on our benchmarks because + /// `bool` (which doesn't care about errors) is used heavily when evaluating statically known branches. + /// + /// An alternative to this flag is to implement a trait similar to Rust's `Try` trait. + /// The advantage of that is that it would allow collecting the errors as well. However, + /// it is significantly more complex and duplicating the logic into `bool` without the error + /// handling didn't show any significant performance difference to when using the `allow_short_circuit` flag. + #[inline] + fn try_bool_impl( + &self, + db: &'db dyn Db, + allow_short_circuit: bool, + visitor: &TryBoolVisitor<'db>, + ) -> Result> { + let type_to_truthiness = |ty: Type<'db>| { + match ty.as_literal_value_kind() { + Some(LiteralValueTypeKind::Bool(bool_val)) => Truthiness::from(bool_val), + Some(LiteralValueTypeKind::Int(int_val)) => Truthiness::from(int_val.as_i64() != 0), + // anything else is handled lower down + _ => Truthiness::Ambiguous, + } + }; + + let try_dunders = || { + match self.try_call_dunder( + db, + "__bool__", + CallArguments::none(), + TypeContext::default(), + ) { + Ok(outcome) => { + let return_type = outcome.return_type(db); + if !return_type.is_assignable_to(db, KnownClass::Bool.to_instance(db)) { + // The type has a `__bool__` method, but it doesn't return a + // boolean. + return Err(BoolError::IncorrectReturnType { + return_type, + not_boolable_type: *self, + }); + } + Ok(type_to_truthiness(return_type)) + } + + Err(CallDunderError::PossiblyUnbound(outcome)) => { + let return_type = outcome.return_type(db); + if !return_type.is_assignable_to(db, KnownClass::Bool.to_instance(db)) { + // The type has a `__bool__` method, but it doesn't return a + // boolean. + return Err(BoolError::IncorrectReturnType { + return_type: outcome.return_type(db), + not_boolable_type: *self, + }); + } + + // Don't trust possibly missing `__bool__` method. + Ok(Truthiness::Ambiguous) + } + + Err(CallDunderError::MethodNotAvailable) => { + // We only consider `__len__` for tuples and `@final` types, + // since `__bool__` takes precedence + // and a subclass could add a `__bool__` method. + // + // TODO: with regards to tuple types, we intend to emit a diagnostic + // if a tuple subclass defines a `__bool__` method with a return type + // that is inconsistent with the tuple's length. Otherwise, the special + // handling for tuples here isn't sound. + if let Some(instance) = self.as_nominal_instance() { + if let Some(tuple_spec) = instance.tuple_spec(db) { + Ok(tuple_spec.truthiness()) + } else if instance.class(db).is_final(db) { + match self.try_call_dunder( + db, + "__len__", + CallArguments::none(), + TypeContext::default(), + ) { + Ok(outcome) => { + let return_type = outcome.return_type(db); + if return_type.is_assignable_to( + db, + KnownClass::SupportsIndex.to_instance(db), + ) { + Ok(type_to_truthiness(return_type)) + } else { + // TODO: should report a diagnostic similar to if return type of `__bool__` + // is not assignable to `bool` + Ok(Truthiness::Ambiguous) + } + } + // if a `@final` type does not define `__bool__` or `__len__`, it is always truthy + Err(CallDunderError::MethodNotAvailable) => { + Ok(Truthiness::AlwaysTrue) + } + // TODO: errors during a `__len__` call (if `__len__` exists) should be reported + // as diagnostics similar to errors during a `__bool__` call (when `__bool__` exists) + Err(_) => Ok(Truthiness::Ambiguous), + } + } else { + Ok(Truthiness::Ambiguous) + } + } else { + Ok(Truthiness::Ambiguous) + } + } + + Err(CallDunderError::CallError(CallErrorKind::BindingError, bindings)) => { + Err(BoolError::IncorrectArguments { + truthiness: type_to_truthiness(bindings.return_type(db)), + not_boolable_type: *self, + }) + } + + Err(CallDunderError::CallError(CallErrorKind::NotCallable, _)) => { + Err(BoolError::NotCallable { + not_boolable_type: *self, + }) + } + + Err(CallDunderError::CallError(CallErrorKind::PossiblyNotCallable, _)) => { + Err(BoolError::Other { + not_boolable_type: *self, + }) + } + } + }; + + let try_union = |union: UnionType<'db>| { + let mut truthiness = None; + let mut all_not_callable = true; + let mut has_errors = false; + + for element in union.elements(db) { + let element_truthiness = + match element.try_bool_impl(db, allow_short_circuit, visitor) { + Ok(truthiness) => truthiness, + Err(err) => { + has_errors = true; + all_not_callable &= matches!(err, BoolError::NotCallable { .. }); + err.fallback_truthiness() + } + }; + + truthiness.get_or_insert(element_truthiness); + + if Some(element_truthiness) != truthiness { + truthiness = Some(Truthiness::Ambiguous); + + if allow_short_circuit { + return Ok(Truthiness::Ambiguous); + } + } + } + + if has_errors { + if all_not_callable { + return Err(BoolError::NotCallable { + not_boolable_type: *self, + }); + } + return Err(BoolError::Union { + union, + truthiness: truthiness.unwrap_or(Truthiness::Ambiguous), + }); + } + Ok(truthiness.unwrap_or(Truthiness::Ambiguous)) + }; + + let truthiness = match self { + Type::Dynamic(_) + | Type::Never + | Type::Callable(_) + | Type::TypeIs(_) + | Type::TypeGuard(_) => Truthiness::Ambiguous, + + Type::TypedDict(td) => { + if td.items(db).values().any(TypedDictField::is_required) { + Truthiness::AlwaysTrue + } else { + // We can potentially infer empty typeddicts as always falsy if they're `closed=True`, + // but as of 22-01-26 we don't yet support PEP 728. + Truthiness::Ambiguous + } + } + + Type::KnownInstance(KnownInstanceType::ConstraintSet(tracked_set)) => { + let constraints = ConstraintSetBuilder::new(); + let tracked_set = constraints.load(tracked_set.constraints(db)); + Truthiness::from(tracked_set.is_always_satisfied(db)) + } + + Type::FunctionLiteral(_) + | Type::BoundMethod(_) + | Type::WrapperDescriptor(_) + | Type::KnownBoundMethod(_) + | Type::DataclassDecorator(_) + | Type::DataclassTransformer(_) + | Type::ModuleLiteral(_) + | Type::PropertyInstance(_) + | Type::BoundSuper(_) + | Type::KnownInstance(_) + | Type::SpecialForm(_) + | Type::AlwaysTruthy => Truthiness::AlwaysTrue, + + Type::AlwaysFalsy => Truthiness::AlwaysFalse, + + Type::ClassLiteral(class) => { + class + .metaclass_instance_type(db) + .try_bool_impl(db, allow_short_circuit, visitor)? + } + Type::GenericAlias(alias) => ClassType::from(*alias) + .metaclass_instance_type(db) + .try_bool_impl(db, allow_short_circuit, visitor)?, + + Type::SubclassOf(subclass_of_ty) => { + match subclass_of_ty.subclass_of().with_transposed_type_var(db) { + SubclassOfInner::Dynamic(_) => Truthiness::Ambiguous, + SubclassOfInner::Class(class) => { + Type::from(class).try_bool_impl(db, allow_short_circuit, visitor)? + } + SubclassOfInner::TypeVar(bound_typevar) => Type::TypeVar(bound_typevar) + .try_bool_impl(db, allow_short_circuit, visitor)?, + } + } + + Type::TypeVar(bound_typevar) => { + match bound_typevar.typevar(db).bound_or_constraints(db) { + None => Truthiness::Ambiguous, + Some(TypeVarBoundOrConstraints::UpperBound(bound)) => { + bound.try_bool_impl(db, allow_short_circuit, visitor)? + } + Some(TypeVarBoundOrConstraints::Constraints(constraints)) => constraints + .as_type(db) + .try_bool_impl(db, allow_short_circuit, visitor)?, + } + } + + Type::NominalInstance(instance) => instance + .known_class(db) + .and_then(KnownClass::bool) + .map(Ok) + .unwrap_or_else(try_dunders)?, + + Type::ProtocolInstance(_) => try_dunders()?, + + Type::Union(union) => try_union(*union)?, + + Type::Intersection(_) => { + // TODO + Truthiness::Ambiguous + } + + Type::LiteralValue(literal) => match literal.kind() { + LiteralValueTypeKind::LiteralString => Truthiness::Ambiguous, + LiteralValueTypeKind::Enum(enum_type) => enum_type + .enum_class_instance(db) + .try_bool_impl(db, allow_short_circuit, visitor)?, + + LiteralValueTypeKind::Int(num) => Truthiness::from(num.as_i64() != 0), + LiteralValueTypeKind::Bool(bool) => Truthiness::from(bool), + LiteralValueTypeKind::String(str) => Truthiness::from(!str.value(db).is_empty()), + LiteralValueTypeKind::Bytes(bytes) => Truthiness::from(!bytes.value(db).is_empty()), + }, + + Type::TypeAlias(alias) => visitor.visit(*self, || { + alias + .value_type(db) + .try_bool_impl(db, allow_short_circuit, visitor) + })?, + Type::NewTypeInstance(newtype) => { + newtype + .concrete_base_type(db) + .try_bool_impl(db, allow_short_circuit, visitor)? + } + }; + + Ok(truthiness) + } +} + +/// A [`CycleDetector`] that is used in `try_bool` methods. +pub(crate) type TryBoolVisitor<'db> = + CycleDetector, Result>>; +pub(crate) struct TryBool; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum BoolError<'db> { + /// The type has a `__bool__` attribute but it can't be called. + NotCallable { not_boolable_type: Type<'db> }, + + /// The type has a callable `__bool__` attribute, but it isn't callable + /// with the given arguments. + IncorrectArguments { + not_boolable_type: Type<'db>, + truthiness: Truthiness, + }, + + /// The type has a `__bool__` method, is callable with the given arguments, + /// but the return type isn't assignable to `bool`. + IncorrectReturnType { + not_boolable_type: Type<'db>, + return_type: Type<'db>, + }, + + /// A union type doesn't implement `__bool__` correctly. + Union { + union: UnionType<'db>, + truthiness: Truthiness, + }, + + /// Any other reason why the type can't be converted to a bool. + /// E.g. because calling `__bool__` returns in a union type and not all variants support `__bool__` or + /// because `__bool__` points to a type that has a possibly missing `__call__` method. + Other { not_boolable_type: Type<'db> }, +} + +impl<'db> BoolError<'db> { + pub(super) fn fallback_truthiness(&self) -> Truthiness { + match self { + BoolError::NotCallable { .. } + | BoolError::IncorrectReturnType { .. } + | BoolError::Other { .. } => Truthiness::Ambiguous, + BoolError::IncorrectArguments { truthiness, .. } + | BoolError::Union { truthiness, .. } => *truthiness, + } + } + + fn not_boolable_type(&self) -> Type<'db> { + match self { + BoolError::NotCallable { + not_boolable_type, .. + } + | BoolError::IncorrectArguments { + not_boolable_type, .. + } + | BoolError::Other { not_boolable_type } + | BoolError::IncorrectReturnType { + not_boolable_type, .. + } => *not_boolable_type, + BoolError::Union { union, .. } => Type::Union(*union), + } + } + + pub(super) fn report_diagnostic(&self, context: &InferContext, condition: impl Ranged) { + self.report_diagnostic_impl(context, condition.range()); + } + + fn report_diagnostic_impl(&self, context: &InferContext, condition: TextRange) { + let Some(builder) = context.report_lint(&UNSUPPORTED_BOOL_CONVERSION, condition) else { + return; + }; + match self { + Self::IncorrectArguments { + not_boolable_type, .. + } => { + let mut diag = builder.into_diagnostic(format_args!( + "Boolean conversion is not supported for type `{}`", + not_boolable_type.display(context.db()) + )); + let mut sub = SubDiagnostic::new( + SubDiagnosticSeverity::Info, + "`__bool__` methods must only have a `self` parameter", + ); + if let Some((func_span, parameter_span)) = not_boolable_type + .member(context.db(), "__bool__") + .into_lookup_result(context.db()) + .ok() + .and_then(|quals| quals.inner_type().parameter_span(context.db(), None)) + { + sub.annotate( + Annotation::primary(parameter_span).message("Incorrect parameters"), + ); + sub.annotate(Annotation::secondary(func_span).message("Method defined here")); + } + diag.sub(sub); + } + Self::IncorrectReturnType { + not_boolable_type, + return_type, + } => { + let mut diag = builder.into_diagnostic(format_args!( + "Boolean conversion is not supported for type `{not_boolable}`", + not_boolable = not_boolable_type.display(context.db()), + )); + let mut sub = SubDiagnostic::new( + SubDiagnosticSeverity::Info, + format_args!( + "`{return_type}` is not assignable to `bool`", + return_type = return_type.display(context.db()), + ), + ); + if let Some((func_span, return_type_span)) = not_boolable_type + .member(context.db(), "__bool__") + .into_lookup_result(context.db()) + .ok() + .and_then(|quals| quals.inner_type().function_spans(context.db())) + .and_then(|spans| Some((spans.name, spans.return_type?))) + { + sub.annotate( + Annotation::primary(return_type_span).message("Incorrect return type"), + ); + sub.annotate(Annotation::secondary(func_span).message("Method defined here")); + } + diag.sub(sub); + } + Self::NotCallable { not_boolable_type } => { + let mut diag = builder.into_diagnostic(format_args!( + "Boolean conversion is not supported for type `{}`", + not_boolable_type.display(context.db()) + )); + let sub = SubDiagnostic::new( + SubDiagnosticSeverity::Info, + format_args!( + "`__bool__` on `{}` must be callable", + not_boolable_type.display(context.db()) + ), + ); + // TODO: It would be nice to create an annotation here for + // where `__bool__` is defined. At time of writing, I couldn't + // figure out a straight-forward way of doing this. ---AG + diag.sub(sub); + } + Self::Union { union, .. } => { + let first_error = union + .elements(context.db()) + .iter() + .find_map(|element| element.try_bool(context.db()).err()) + .unwrap(); + + builder.into_diagnostic(format_args!( + "Boolean conversion is not supported for union `{}` \ + because `{}` doesn't implement `__bool__` correctly", + Type::Union(*union).display(context.db()), + first_error.not_boolable_type().display(context.db()), + )); + } + + Self::Other { not_boolable_type } => { + builder.into_diagnostic(format_args!( + "Boolean conversion is not supported for type `{}`; \ + it incorrectly implements `__bool__`", + not_boolable_type.display(context.db()) + )); + } + } + } +} diff --git a/crates/ty_python_semantic/src/types/call/bind.rs b/crates/ty_python_semantic/src/types/call/bind.rs index 152847048c90d..053ccbb40c39b 100644 --- a/crates/ty_python_semantic/src/types/call/bind.rs +++ b/crates/ty_python_semantic/src/types/call/bind.rs @@ -45,13 +45,12 @@ use crate::types::signatures::{Parameter, ParameterForm, ParameterKind, Paramete use crate::types::tuple::{TupleLength, TupleSpec, TupleType}; use crate::types::{ BoundMethodType, BoundTypeVarIdentity, BoundTypeVarInstance, CallableSignature, CallableType, - CallableTypeKind, ClassLiteral, DATACLASS_FLAGS, DataclassFlags, DataclassParams, GenericAlias, - InternedConstraintSet, IntersectionType, KnownBoundMethodType, KnownClass, KnownInstanceType, - LiteralValueTypeKind, MemberLookupPolicy, NominalInstanceType, PropertyInstanceType, - SpecialFormType, TypeAliasType, TypeContext, TypeVarBoundOrConstraints, TypeVarVariance, - UnionBuilder, UnionType, WrapperDescriptorKind, enums, list_members, + CallableTypeKind, ClassLiteral, DATACLASS_FLAGS, DataclassFlags, DataclassParams, + EvaluationMode, GenericAlias, InternedConstraintSet, IntersectionType, KnownBoundMethodType, + KnownClass, KnownInstanceType, LiteralValueTypeKind, MemberLookupPolicy, NominalInstanceType, + PropertyInstanceType, SpecialFormType, TypeAliasType, TypeContext, TypeVarBoundOrConstraints, + TypeVarVariance, UnionBuilder, UnionType, WrapperDescriptorKind, enums, list_members, }; -use crate::unpack::EvaluationMode; use crate::{DisplaySettings, Program}; use ruff_db::diagnostic::{Annotation, Diagnostic, SubDiagnostic, SubDiagnosticSeverity}; use ruff_python_ast::{self as ast, ArgOrKeyword, PythonVersion}; diff --git a/crates/ty_python_semantic/src/types/context_manager.rs b/crates/ty_python_semantic/src/types/context_manager.rs new file mode 100644 index 0000000000000..5ae29c83b48ef --- /dev/null +++ b/crates/ty_python_semantic/src/types/context_manager.rs @@ -0,0 +1,257 @@ +use crate::{ + Db, + types::{ + CallArguments, CallDunderError, EvaluationMode, Type, TypeContext, call::CallErrorKind, + context::InferContext, diagnostic::INVALID_CONTEXT_MANAGER, + }, +}; +use ruff_python_ast as ast; + +impl<'db> Type<'db> { + /// Returns the type bound from a context manager with type `self`. + /// + /// This method should only be used outside of type checking because it omits any errors. + /// For type checking, use [`try_enter_with_mode`](Self::try_enter_with_mode) instead. + pub(super) fn enter(self, db: &'db dyn Db) -> Type<'db> { + self.try_enter_with_mode(db, EvaluationMode::Sync) + .unwrap_or_else(|err| err.fallback_enter_type(db)) + } + + /// Returns the type bound from a context manager with type `self`. + /// + /// This method should only be used outside of type checking because it omits any errors. + /// For type checking, use [`try_enter_with_mode`](Self::try_enter_with_mode) instead. + pub(super) fn aenter(self, db: &'db dyn Db) -> Type<'db> { + self.try_enter_with_mode(db, EvaluationMode::Async) + .unwrap_or_else(|err| err.fallback_enter_type(db)) + } + + /// Given the type of an object that is used as a context manager (i.e. in a `with` statement), + /// return the return type of its `__enter__` or `__aenter__` method, which is bound to any potential targets. + /// + /// E.g., for the following `with` statement, given the type of `x`, infer the type of `y`: + /// ```python + /// with x as y: + /// pass + /// ``` + pub(super) fn try_enter_with_mode( + self, + db: &'db dyn Db, + mode: EvaluationMode, + ) -> Result, ContextManagerError<'db>> { + let (enter_method, exit_method) = match mode { + EvaluationMode::Async => ("__aenter__", "__aexit__"), + EvaluationMode::Sync => ("__enter__", "__exit__"), + }; + + let enter = self.try_call_dunder( + db, + enter_method, + CallArguments::none(), + TypeContext::default(), + ); + let exit = self.try_call_dunder( + db, + exit_method, + CallArguments::positional([Type::none(db), Type::none(db), Type::none(db)]), + TypeContext::default(), + ); + + // TODO: Make use of Protocols when we support it (the manager be assignable to `contextlib.AbstractContextManager`). + match (enter, exit) { + (Ok(enter), Ok(_)) => { + let ty = enter.return_type(db); + Ok(if mode.is_async() { + ty.try_await(db).unwrap_or(Type::unknown()) + } else { + ty + }) + } + (Ok(enter), Err(exit_error)) => { + let ty = enter.return_type(db); + Err(ContextManagerError::Exit { + enter_return_type: if mode.is_async() { + ty.try_await(db).unwrap_or(Type::unknown()) + } else { + ty + }, + exit_error, + mode, + }) + } + // TODO: Use the `exit_ty` to determine if any raised exception is suppressed. + (Err(enter_error), Ok(_)) => Err(ContextManagerError::Enter(enter_error, mode)), + (Err(enter_error), Err(exit_error)) => Err(ContextManagerError::EnterAndExit { + enter_error, + exit_error, + mode, + }), + } + } +} + +/// Error returned if a type is not (or may not be) a context manager. +#[derive(Debug)] +pub(super) enum ContextManagerError<'db> { + Enter(CallDunderError<'db>, EvaluationMode), + Exit { + enter_return_type: Type<'db>, + exit_error: CallDunderError<'db>, + mode: EvaluationMode, + }, + EnterAndExit { + enter_error: CallDunderError<'db>, + exit_error: CallDunderError<'db>, + mode: EvaluationMode, + }, +} + +impl<'db> ContextManagerError<'db> { + pub(super) fn fallback_enter_type(&self, db: &'db dyn Db) -> Type<'db> { + self.enter_type(db).unwrap_or(Type::unknown()) + } + + /// Returns the `__enter__` or `__aenter__` return type if it is known, + /// or `None` if the type never has a callable `__enter__` or `__aenter__` attribute + fn enter_type(&self, db: &'db dyn Db) -> Option> { + match self { + Self::Exit { + enter_return_type, + exit_error: _, + mode: _, + } => Some(*enter_return_type), + Self::Enter(enter_error, _) + | Self::EnterAndExit { + enter_error, + exit_error: _, + mode: _, + } => match enter_error { + CallDunderError::PossiblyUnbound(call_outcome) => { + Some(call_outcome.return_type(db)) + } + CallDunderError::CallError(CallErrorKind::NotCallable, _) => None, + CallDunderError::CallError(_, bindings) => Some(bindings.return_type(db)), + CallDunderError::MethodNotAvailable => None, + }, + } + } + + pub(super) fn report_diagnostic( + &self, + context: &InferContext<'db, '_>, + context_expression_type: Type<'db>, + context_expression_node: ast::AnyNodeRef, + ) { + let Some(builder) = context.report_lint(&INVALID_CONTEXT_MANAGER, context_expression_node) + else { + return; + }; + + let mode = match self { + Self::Exit { mode, .. } | Self::Enter(_, mode) | Self::EnterAndExit { mode, .. } => { + *mode + } + }; + + let (enter_method, exit_method) = match mode { + EvaluationMode::Async => ("__aenter__", "__aexit__"), + EvaluationMode::Sync => ("__enter__", "__exit__"), + }; + + let format_call_dunder_error = |call_dunder_error: &CallDunderError<'db>, name: &str| { + match call_dunder_error { + CallDunderError::MethodNotAvailable => format!("it does not implement `{name}`"), + CallDunderError::PossiblyUnbound(_) => { + format!("the method `{name}` may be missing") + } + // TODO: Use more specific error messages for the different error cases. + // E.g. hint toward the union variant that doesn't correctly implement enter, + // distinguish between a not callable `__enter__` attribute and a wrong signature. + CallDunderError::CallError(_, _) => { + format!("it does not correctly implement `{name}`") + } + } + }; + + let format_call_dunder_errors = |error_a: &CallDunderError<'db>, + name_a: &str, + error_b: &CallDunderError<'db>, + name_b: &str| { + match (error_a, error_b) { + (CallDunderError::PossiblyUnbound(_), CallDunderError::PossiblyUnbound(_)) => { + format!("the methods `{name_a}` and `{name_b}` are possibly missing") + } + (CallDunderError::MethodNotAvailable, CallDunderError::MethodNotAvailable) => { + format!("it does not implement `{name_a}` and `{name_b}`") + } + (CallDunderError::CallError(_, _), CallDunderError::CallError(_, _)) => { + format!("it does not correctly implement `{name_a}` or `{name_b}`") + } + (_, _) => format!( + "{format_a}, and {format_b}", + format_a = format_call_dunder_error(error_a, name_a), + format_b = format_call_dunder_error(error_b, name_b) + ), + } + }; + + let db = context.db(); + + let formatted_errors = match self { + Self::Exit { + enter_return_type: _, + exit_error, + mode: _, + } => format_call_dunder_error(exit_error, exit_method), + Self::Enter(enter_error, _) => format_call_dunder_error(enter_error, enter_method), + Self::EnterAndExit { + enter_error, + exit_error, + mode: _, + } => format_call_dunder_errors(enter_error, enter_method, exit_error, exit_method), + }; + + // Suggest using `async with` if only async methods are available in a sync context, + // or suggest using `with` if only sync methods are available in an async context. + let with_kw = match mode { + EvaluationMode::Sync => "with", + EvaluationMode::Async => "async with", + }; + + let mut diag = builder.into_diagnostic(format_args!( + "Object of type `{}` cannot be used with `{}` because {}", + context_expression_type.display(db), + with_kw, + formatted_errors, + )); + + let (alt_mode, alt_enter_method, alt_exit_method, alt_with_kw) = match mode { + EvaluationMode::Sync => ("async", "__aenter__", "__aexit__", "async with"), + EvaluationMode::Async => ("sync", "__enter__", "__exit__", "with"), + }; + + let alt_enter = context_expression_type.try_call_dunder( + db, + alt_enter_method, + CallArguments::none(), + TypeContext::default(), + ); + let alt_exit = context_expression_type.try_call_dunder( + db, + alt_exit_method, + CallArguments::positional([Type::unknown(), Type::unknown(), Type::unknown()]), + TypeContext::default(), + ); + + if (alt_enter.is_ok() || matches!(alt_enter, Err(CallDunderError::CallError(..)))) + && (alt_exit.is_ok() || matches!(alt_exit, Err(CallDunderError::CallError(..)))) + { + diag.info(format_args!( + "Objects of type `{}` can be used as {} context managers", + context_expression_type.display(db), + alt_mode + )); + diag.info(format!("Consider using `{alt_with_kw}` here")); + } + } +} diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index 243e3e8a1c9e2..0864c40f87d92 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -129,8 +129,8 @@ use crate::types::typed_dict::{ use crate::types::visitor::find_over_type; use crate::types::{ BoundTypeVarIdentity, CallDunderError, CallableBinding, CallableType, CallableTypeKind, - ClassType, DataclassParams, DynamicType, GenericAlias, InternedConstraintSet, InternedType, - IntersectionBuilder, IntersectionType, KnownClass, KnownInstanceType, KnownUnion, + ClassType, DataclassParams, DynamicType, EvaluationMode, GenericAlias, InternedConstraintSet, + InternedType, IntersectionBuilder, IntersectionType, KnownClass, KnownInstanceType, KnownUnion, LintDiagnosticGuard, LiteralValueTypeKind, ManualPEP695TypeAliasType, MemberLookupPolicy, MetaclassCandidate, PEP695TypeAliasType, ParamSpecAttrKind, Parameter, ParameterForm, Parameters, Signature, SpecialFormType, StaticClassLiteral, SubclassOfType, Truthiness, Type, @@ -142,7 +142,7 @@ use crate::types::{ }; use crate::types::{CallableTypes, overrides}; use crate::types::{ClassBase, add_inferred_python_version_hint_to_diagnostic}; -use crate::unpack::{EvaluationMode, UnpackPosition}; +use crate::unpack::UnpackPosition; use crate::{AnalysisSettings, Db, FxIndexSet, Program}; mod annotation_expression; diff --git a/crates/ty_python_semantic/src/types/iteration.rs b/crates/ty_python_semantic/src/types/iteration.rs new file mode 100644 index 0000000000000..3b4ad5d5441a7 --- /dev/null +++ b/crates/ty_python_semantic/src/types/iteration.rs @@ -0,0 +1,822 @@ +use crate::{ + Db, + types::{ + AwaitError, Bindings, CallArguments, CallDunderError, EvaluationMode, KnownClass, + LintDiagnosticGuard, LintDiagnosticGuardBuilder, LiteralValueTypeKind, Type, TypeContext, + TypeVarBoundOrConstraints, UnionType, + call::CallErrorKind, + context::InferContext, + diagnostic::NOT_ITERABLE, + todo_type, + tuple::{TupleSpec, TupleSpecBuilder}, + }, +}; +use ruff_python_ast as ast; +use std::borrow::Cow; + +impl<'db> Type<'db> { + /// Returns a tuple spec describing the elements that are produced when iterating over `self`. + /// + /// This method should only be used outside of type checking because it omits any errors. + /// For type checking, use [`try_iterate`](Self::try_iterate) instead. + pub(super) fn iterate(self, db: &'db dyn Db) -> Cow<'db, TupleSpec<'db>> { + self.try_iterate(db) + .unwrap_or_else(|err| Cow::Owned(TupleSpec::homogeneous(err.fallback_element_type(db)))) + } + + /// Given the type of an object that is iterated over in some way, + /// return a tuple spec describing the type of objects that are yielded by that iteration. + /// + /// E.g., for the following call, given the type of `x`, infer the types of the values that are + /// splatted into `y`'s positional arguments: + /// ```python + /// y(*x) + /// ``` + pub(super) fn try_iterate( + self, + db: &'db dyn Db, + ) -> Result>, IterationError<'db>> { + self.try_iterate_with_mode(db, EvaluationMode::Sync) + } + + pub(super) fn try_iterate_with_mode( + self, + db: &'db dyn Db, + mode: EvaluationMode, + ) -> Result>, IterationError<'db>> { + fn non_async_special_case<'db>( + db: &'db dyn Db, + ty: Type<'db>, + ) -> Option>> { + // We will not infer precise heterogeneous tuple specs for literals with lengths above this threshold. + // The threshold here is somewhat arbitrary and conservative; it could be increased if needed. + // However, it's probably very rare to need heterogeneous unpacking inference for long string literals + // or bytes literals, and creating long heterogeneous tuple specs has a performance cost. + const MAX_TUPLE_LENGTH: usize = 128; + + match ty { + Type::NominalInstance(nominal) => nominal.tuple_spec(db), + Type::NewTypeInstance(newtype) => non_async_special_case(db, newtype.concrete_base_type(db)), + Type::GenericAlias(alias) if alias.origin(db).is_tuple(db) => { + Some(Cow::Owned(TupleSpec::homogeneous(todo_type!( + "*tuple[] annotations" + )))) + } + Type::LiteralValue(literal) => match literal.kind() { + LiteralValueTypeKind::Bytes(bytes) => { + let bytes_literal = bytes.value(db); + let spec = if bytes_literal.len() < MAX_TUPLE_LENGTH { + TupleSpec::heterogeneous( + bytes_literal + .iter() + .map(|b| Type::int_literal( i64::from(*b))), + ) + } else { + TupleSpec::homogeneous(KnownClass::Int.to_instance(db)) + }; + Some(Cow::Owned(spec)) + }, + LiteralValueTypeKind::String(string_literal_ty) => { + let string_literal = string_literal_ty.value(db); + let spec = if string_literal.len() < MAX_TUPLE_LENGTH { + TupleSpec::heterogeneous( + string_literal + .chars() + .map(|c| Type::string_literal(db, &c.to_string())), + ) + } else { + TupleSpec::homogeneous(Type::literal_string()) + }; + Some(Cow::Owned(spec)) + } + // N.B. This special case isn't strictly necessary, it's just an obvious optimization + LiteralValueTypeKind::LiteralString => { + Some(Cow::Owned(TupleSpec::homogeneous(ty))) + } + _ => None + } + Type::Never => { + // The dunder logic below would have us return `tuple[Never, ...]`, which eagerly + // simplifies to `tuple[()]`. That will will cause us to emit false positives if we + // index into the tuple. Using `tuple[Unknown, ...]` avoids these false positives. + // TODO: Consider removing this special case, and instead hide the indexing + // diagnostic in unreachable code. + Some(Cow::Owned(TupleSpec::homogeneous(Type::unknown()))) + } + Type::TypeAlias(alias) => { + non_async_special_case(db, alias.value_type(db)) + } + Type::TypeVar(tvar) => match tvar.typevar(db).bound_or_constraints(db)? { + TypeVarBoundOrConstraints::UpperBound(bound) => { + non_async_special_case(db, bound) + } + TypeVarBoundOrConstraints::Constraints(constraints) => non_async_special_case(db, constraints.as_type(db)), + }, + Type::Union(union) => { + let elements = union.elements(db); + if elements.len() < MAX_TUPLE_LENGTH { + let mut elements_iter = elements.iter(); + let first_element_spec = elements_iter.next()?.try_iterate_with_mode(db, EvaluationMode::Sync).ok()?; + let mut builder = TupleSpecBuilder::from(&*first_element_spec); + for element in elements_iter { + builder = builder.union(db, &*element.try_iterate_with_mode(db, EvaluationMode::Sync).ok()?); + } + Some(Cow::Owned(builder.build())) + } else { + None + } + } + Type::Intersection(intersection) => { + // For intersections containing TypeVars with union bounds, we need to + // flatten the TypeVars first. This distributes the intersection over + // the union and simplifies, e.g.: + // `T & tuple[object, ...]` where `T: tuple[int, ...] | list[str]` + // becomes `(tuple[int, ...] & tuple[object, ...]) | (list[str] & tuple[object, ...])` + // which simplifies to `tuple[int, ...] | Never` = `tuple[int, ...]` + // + // After flattening, the result may be: + // - An intersection (if no union-bound typevars, or they didn't simplify). + // - A union of intersections (if distribution happened). + // - A simpler type (if it fully simplified). + // + // We then iterate over the flattened type. + let flattened = ty.flatten_typevars(db); + + // If flattening didn't change anything, iterate the intersection directly. + if flattened == ty { + let mut specs_iter = intersection.positive_elements_or_object(db).filter_map( + |element| element.try_iterate_with_mode(db, EvaluationMode::Sync).ok(), + ); + let first_spec = specs_iter.next()?; + let mut builder = TupleSpecBuilder::from(&*first_spec); + for spec in specs_iter { + // Two tuples cannot have incompatible specs unless the tuples themselves + // are disjoint. `IntersectionBuilder` eagerly simplifies such + // intersections to `Never`, so this should always return `Some`. + let Some(intersected) = builder.intersect(db, &spec) else { + return Some(Cow::Owned(TupleSpec::homogeneous(Type::unknown()))); + }; + builder = intersected; + } + return Some(Cow::Owned(builder.build())); + } + + // Flattening changed the type; recursively iterate the flattened result. + non_async_special_case(db, flattened) + } + // N.B. This special case isn't strictly necessary, it's just an obvious optimization + Type::Dynamic(_) => Some(Cow::Owned(TupleSpec::homogeneous(ty))), + + Type::FunctionLiteral(_) + | Type::GenericAlias(_) + | Type::BoundMethod(_) + | Type::KnownBoundMethod(_) + | Type::WrapperDescriptor(_) + | Type::DataclassDecorator(_) + | Type::DataclassTransformer(_) + | Type::Callable(_) + | Type::ModuleLiteral(_) + // We could infer a precise tuple spec for enum classes with members, + // but it's not clear whether that's worth the added complexity: + // you'd have to check that `EnumMeta.__iter__` is not overridden for it to be sound + // (enums can have `EnumMeta` subclasses as their metaclasses). + | Type::ClassLiteral(_) + | Type::SubclassOf(_) + | Type::ProtocolInstance(_) + | Type::SpecialForm(_) + | Type::KnownInstance(_) + | Type::PropertyInstance(_) + | Type::AlwaysTruthy + | Type::AlwaysFalsy + | Type::BoundSuper(_) + | Type::TypeIs(_) + | Type::TypeGuard(_) + | Type::TypedDict(_) => None + } + } + + if mode.is_async() { + let try_call_dunder_anext_on_iterator = |iterator: Type<'db>| -> Result< + Result, AwaitError<'db>>, + CallDunderError<'db>, + > { + iterator + .try_call_dunder( + db, + "__anext__", + CallArguments::none(), + TypeContext::default(), + ) + .map(|dunder_anext_outcome| dunder_anext_outcome.return_type(db).try_await(db)) + }; + + return match self.try_call_dunder( + db, + "__aiter__", + CallArguments::none(), + TypeContext::default(), + ) { + Ok(dunder_aiter_bindings) => { + let iterator = dunder_aiter_bindings.return_type(db); + match try_call_dunder_anext_on_iterator(iterator) { + Ok(Ok(result)) => Ok(Cow::Owned(TupleSpec::homogeneous(result))), + Ok(Err(AwaitError::InvalidReturnType(..))) => { + Err(IterationError::UnboundAiterError) + } // TODO: __anext__ is bound, but is not properly awaitable + Err(dunder_anext_error) | Ok(Err(AwaitError::Call(dunder_anext_error))) => { + Err(IterationError::IterReturnsInvalidIterator { + iterator, + dunder_error: dunder_anext_error, + mode, + }) + } + } + } + Err(CallDunderError::PossiblyUnbound(dunder_aiter_bindings)) => { + let iterator = dunder_aiter_bindings.return_type(db); + match try_call_dunder_anext_on_iterator(iterator) { + Ok(_) => Err(IterationError::IterCallError { + kind: CallErrorKind::PossiblyNotCallable, + bindings: dunder_aiter_bindings, + mode, + }), + Err(dunder_anext_error) => { + Err(IterationError::IterReturnsInvalidIterator { + iterator, + dunder_error: dunder_anext_error, + mode, + }) + } + } + } + Err(CallDunderError::CallError(kind, bindings)) => { + Err(IterationError::IterCallError { + kind, + bindings, + mode, + }) + } + Err(CallDunderError::MethodNotAvailable) => Err(IterationError::UnboundAiterError), + }; + } + + if let Some(special_case) = non_async_special_case(db, self) { + return Ok(special_case); + } + + let try_call_dunder_getitem = || { + self.try_call_dunder( + db, + "__getitem__", + CallArguments::positional([KnownClass::Int.to_instance(db)]), + TypeContext::default(), + ) + .map(|dunder_getitem_outcome| dunder_getitem_outcome.return_type(db)) + }; + + let try_call_dunder_next_on_iterator = |iterator: Type<'db>| { + iterator + .try_call_dunder( + db, + "__next__", + CallArguments::none(), + TypeContext::default(), + ) + .map(|dunder_next_outcome| dunder_next_outcome.return_type(db)) + }; + + let dunder_iter_result = self + .try_call_dunder( + db, + "__iter__", + CallArguments::none(), + TypeContext::default(), + ) + .map(|dunder_iter_outcome| dunder_iter_outcome.return_type(db)); + + match dunder_iter_result { + Ok(iterator) => { + // `__iter__` is definitely bound and calling it succeeds. + // See what calling `__next__` on the object returned by `__iter__` gives us... + try_call_dunder_next_on_iterator(iterator) + .map(|ty| Cow::Owned(TupleSpec::homogeneous(ty))) + .map_err( + |dunder_next_error| IterationError::IterReturnsInvalidIterator { + iterator, + dunder_error: dunder_next_error, + mode, + }, + ) + } + + // `__iter__` is possibly unbound... + Err(CallDunderError::PossiblyUnbound(dunder_iter_outcome)) => { + let iterator = dunder_iter_outcome.return_type(db); + + match try_call_dunder_next_on_iterator(iterator) { + Ok(dunder_next_return) => { + try_call_dunder_getitem() + .map(|dunder_getitem_return_type| { + // If `__iter__` is possibly unbound, + // but it returns an object that has a bound and valid `__next__` method, + // *and* the object has a bound and valid `__getitem__` method, + // we infer a union of the type returned by the `__next__` method + // and the type returned by the `__getitem__` method. + // + // No diagnostic is emitted; iteration will always succeed! + Cow::Owned(TupleSpec::homogeneous(UnionType::from_two_elements( + db, + dunder_next_return, + dunder_getitem_return_type, + ))) + }) + .map_err(|dunder_getitem_error| { + IterationError::PossiblyUnboundIterAndGetitemError { + dunder_next_return, + dunder_getitem_error, + } + }) + } + + Err(dunder_next_error) => Err(IterationError::IterReturnsInvalidIterator { + iterator, + dunder_error: dunder_next_error, + mode, + }), + } + } + + // `__iter__` is definitely bound but it can't be called with the expected arguments + Err(CallDunderError::CallError(kind, bindings)) => Err(IterationError::IterCallError { + kind, + bindings, + mode, + }), + + // There's no `__iter__` method. Try `__getitem__` instead... + Err(CallDunderError::MethodNotAvailable) => try_call_dunder_getitem() + .map(|ty| Cow::Owned(TupleSpec::homogeneous(ty))) + .map_err( + |dunder_getitem_error| IterationError::UnboundIterAndGetitemError { + dunder_getitem_error, + }, + ), + } + } +} + +/// Error returned if a type is not (or may not be) iterable. +#[derive(Debug)] +pub(super) enum IterationError<'db> { + /// The object being iterated over has a bound `__(a)iter__` method, + /// but calling it with the expected arguments results in an error. + IterCallError { + kind: CallErrorKind, + bindings: Box>, + mode: EvaluationMode, + }, + + /// The object being iterated over has a bound `__(a)iter__` method that can be called + /// with the expected types, but it returns an object that is not a valid iterator. + IterReturnsInvalidIterator { + /// The type of the object returned by the `__(a)iter__` method. + iterator: Type<'db>, + /// The error we encountered when we tried to call `__(a)next__` on the type + /// returned by `__(a)iter__` + dunder_error: CallDunderError<'db>, + /// Whether this is a synchronous or an asynchronous iterator. + mode: EvaluationMode, + }, + + /// The object being iterated over has a bound `__iter__` method that returns a + /// valid iterator. However, the `__iter__` method is possibly unbound, and there + /// either isn't a `__getitem__` method to fall back to, or calling the `__getitem__` + /// method returns some kind of error. + PossiblyUnboundIterAndGetitemError { + /// The type of the object returned by the `__next__` method on the iterator. + /// (The iterator being the type returned by the `__iter__` method on the iterable.) + dunder_next_return: Type<'db>, + /// The error we encountered when we tried to call `__getitem__` on the iterable. + dunder_getitem_error: CallDunderError<'db>, + }, + + /// The object being iterated over doesn't have an `__iter__` method. + /// It also either doesn't have a `__getitem__` method to fall back to, + /// or calling the `__getitem__` method returns some kind of error. + UnboundIterAndGetitemError { + dunder_getitem_error: CallDunderError<'db>, + }, + + /// The asynchronous iterable has no `__aiter__` method. + UnboundAiterError, +} + +impl<'db> IterationError<'db> { + pub(super) fn fallback_element_type(&self, db: &'db dyn Db) -> Type<'db> { + self.element_type(db).unwrap_or(Type::unknown()) + } + + /// Returns the element type if it is known, or `None` if the type is never iterable. + fn element_type(&self, db: &'db dyn Db) -> Option> { + let return_type = |result: Result, CallDunderError<'db>>| { + result + .map(|outcome| Some(outcome.return_type(db))) + .unwrap_or_else(|call_error| call_error.return_type(db)) + }; + + match self { + Self::IterReturnsInvalidIterator { + dunder_error, mode, .. + } => dunder_error.return_type(db).and_then(|ty| { + if mode.is_async() { + ty.try_await(db).ok() + } else { + Some(ty) + } + }), + + Self::IterCallError { + kind: _, + bindings: dunder_iter_bindings, + mode, + } => { + if mode.is_async() { + return_type(dunder_iter_bindings.return_type(db).try_call_dunder( + db, + "__anext__", + CallArguments::none(), + TypeContext::default(), + )) + .and_then(|ty| ty.try_await(db).ok()) + } else { + return_type(dunder_iter_bindings.return_type(db).try_call_dunder( + db, + "__next__", + CallArguments::none(), + TypeContext::default(), + )) + } + } + + Self::PossiblyUnboundIterAndGetitemError { + dunder_next_return, + dunder_getitem_error, + } => match dunder_getitem_error { + CallDunderError::MethodNotAvailable => Some(*dunder_next_return), + CallDunderError::PossiblyUnbound(dunder_getitem_outcome) => { + Some(UnionType::from_two_elements( + db, + *dunder_next_return, + dunder_getitem_outcome.return_type(db), + )) + } + CallDunderError::CallError(CallErrorKind::NotCallable, _) => { + Some(*dunder_next_return) + } + CallDunderError::CallError(_, dunder_getitem_bindings) => { + let dunder_getitem_return = dunder_getitem_bindings.return_type(db); + Some(UnionType::from_two_elements( + db, + *dunder_next_return, + dunder_getitem_return, + )) + } + }, + + Self::UnboundIterAndGetitemError { + dunder_getitem_error, + } => dunder_getitem_error.return_type(db), + + Self::UnboundAiterError => None, + } + } + + /// Does this error concern a synchronous or asynchronous iterable? + fn mode(&self) -> EvaluationMode { + match self { + Self::IterCallError { mode, .. } => *mode, + Self::IterReturnsInvalidIterator { mode, .. } => *mode, + Self::PossiblyUnboundIterAndGetitemError { .. } + | Self::UnboundIterAndGetitemError { .. } => EvaluationMode::Sync, + Self::UnboundAiterError => EvaluationMode::Async, + } + } + + /// Reports the diagnostic for this error. + pub(super) fn report_diagnostic( + &self, + context: &InferContext<'db, '_>, + iterable_type: Type<'db>, + iterable_node: ast::AnyNodeRef, + ) { + /// A little helper type for emitting a diagnostic + /// based on the variant of iteration error. + struct Reporter<'a> { + db: &'a dyn Db, + builder: LintDiagnosticGuardBuilder<'a, 'a>, + iterable_type: Type<'a>, + mode: EvaluationMode, + } + + impl<'a> Reporter<'a> { + /// Emit a diagnostic that is certain that `iterable_type` is not iterable. + /// + /// `because` should explain why `iterable_type` is not iterable. + #[expect(clippy::wrong_self_convention)] + fn is_not(self, because: impl std::fmt::Display) -> LintDiagnosticGuard<'a, 'a> { + let mut diag = self.builder.into_diagnostic(format_args!( + "Object of type `{iterable_type}` is not {maybe_async}iterable", + iterable_type = self.iterable_type.display(self.db), + maybe_async = if self.mode.is_async() { "async-" } else { "" } + )); + diag.info(because); + diag + } + + /// Emit a diagnostic that is uncertain that `iterable_type` is not iterable. + /// + /// `because` should explain why `iterable_type` is likely not iterable. + fn may_not(self, because: impl std::fmt::Display) -> LintDiagnosticGuard<'a, 'a> { + let mut diag = self.builder.into_diagnostic(format_args!( + "Object of type `{iterable_type}` may not be {maybe_async}iterable", + iterable_type = self.iterable_type.display(self.db), + maybe_async = if self.mode.is_async() { "async-" } else { "" } + )); + diag.info(because); + diag + } + } + + let Some(builder) = context.report_lint(&NOT_ITERABLE, iterable_node) else { + return; + }; + let db = context.db(); + let mode = self.mode(); + let reporter = Reporter { + db, + builder, + iterable_type, + mode, + }; + + // TODO: for all of these error variants, the "explanation" for the diagnostic + // (everything after the "because") should really be presented as a "help:", "note", + // or similar, rather than as part of the same sentence as the error message. + match self { + Self::IterCallError { + kind, + bindings, + mode, + } => { + let method = if mode.is_async() { + "__aiter__" + } else { + "__iter__" + }; + + match kind { + CallErrorKind::NotCallable => { + reporter.is_not(format_args!( + "Its `{method}` attribute has type `{dunder_iter_type}`, which is not callable", + dunder_iter_type = bindings.callable_type().display(db), + )); + } + CallErrorKind::PossiblyNotCallable => { + reporter.may_not(format_args!( + "Its `{method}` attribute (with type `{dunder_iter_type}`) \ + may not be callable", + dunder_iter_type = bindings.callable_type().display(db), + )); + } + CallErrorKind::BindingError => { + if bindings.is_single() { + reporter + .is_not(format_args!( + "Its `{method}` method has an invalid signature" + )) + .info(format_args!("Expected signature `def {method}(self): ...`")); + } else { + let mut diag = reporter.may_not(format_args!( + "Its `{method}` method may have an invalid signature" + )); + diag.info(format_args!( + "Type of `{method}` is `{dunder_iter_type}`", + dunder_iter_type = bindings.callable_type().display(db), + )); + diag.info(format_args!( + "Expected signature for `{method}` is `def {method}(self): ...`", + )); + } + } + } + } + + Self::IterReturnsInvalidIterator { + iterator, + dunder_error: dunder_next_error, + mode, + } => { + let dunder_iter_name = if mode.is_async() { + "__aiter__" + } else { + "__iter__" + }; + let dunder_next_name = if mode.is_async() { + "__anext__" + } else { + "__next__" + }; + match dunder_next_error { + CallDunderError::MethodNotAvailable => { + reporter.is_not(format_args!( + "Its `{dunder_iter_name}` method returns an object of type `{iterator_type}`, \ + which has no `{dunder_next_name}` method", + iterator_type = iterator.display(db), + )); + } + CallDunderError::PossiblyUnbound(_) => { + reporter.may_not(format_args!( + "Its `{dunder_iter_name}` method returns an object of type `{iterator_type}`, \ + which may not have a `{dunder_next_name}` method", + iterator_type = iterator.display(db), + )); + } + CallDunderError::CallError(CallErrorKind::NotCallable, _) => { + reporter.is_not(format_args!( + "Its `{dunder_iter_name}` method returns an object of type `{iterator_type}`, \ + which has a `{dunder_next_name}` attribute that is not callable", + iterator_type = iterator.display(db), + )); + } + CallDunderError::CallError(CallErrorKind::PossiblyNotCallable, _) => { + reporter.may_not(format_args!( + "Its `{dunder_iter_name}` method returns an object of type `{iterator_type}`, \ + which has a `{dunder_next_name}` attribute that may not be callable", + iterator_type = iterator.display(db), + )); + } + CallDunderError::CallError(CallErrorKind::BindingError, bindings) + if bindings.is_single() => + { + reporter + .is_not(format_args!( + "Its `{dunder_iter_name}` method returns an object of type `{iterator_type}`, \ + which has an invalid `{dunder_next_name}` method", + iterator_type = iterator.display(db), + )) + .info(format_args!("Expected signature for `{dunder_next_name}` is `def {dunder_next_name}(self): ...`")); + } + CallDunderError::CallError(CallErrorKind::BindingError, _) => { + reporter + .may_not(format_args!( + "Its `{dunder_iter_name}` method returns an object of type `{iterator_type}`, \ + which may have an invalid `{dunder_next_name}` method", + iterator_type = iterator.display(db), + )) + .info(format_args!("Expected signature for `{dunder_next_name}` is `def {dunder_next_name}(self): ...`")); + } + } + } + + Self::PossiblyUnboundIterAndGetitemError { + dunder_getitem_error, + .. + } => match dunder_getitem_error { + CallDunderError::MethodNotAvailable => { + reporter.may_not( + "It may not have an `__iter__` method \ + and it doesn't have a `__getitem__` method", + ); + } + CallDunderError::PossiblyUnbound(_) => { + reporter + .may_not("It may not have an `__iter__` method or a `__getitem__` method"); + } + CallDunderError::CallError(CallErrorKind::NotCallable, bindings) => { + reporter.may_not(format_args!( + "It may not have an `__iter__` method \ + and its `__getitem__` attribute has type `{dunder_getitem_type}`, \ + which is not callable", + dunder_getitem_type = bindings.callable_type().display(db), + )); + } + CallDunderError::CallError(CallErrorKind::PossiblyNotCallable, bindings) + if bindings.is_single() => + { + reporter.may_not( + "It may not have an `__iter__` method \ + and its `__getitem__` attribute may not be callable", + ); + } + CallDunderError::CallError(CallErrorKind::PossiblyNotCallable, bindings) => { + reporter.may_not(format_args!( + "It may not have an `__iter__` method \ + and its `__getitem__` attribute (with type `{dunder_getitem_type}`) \ + may not be callable", + dunder_getitem_type = bindings.callable_type().display(db), + )); + } + CallDunderError::CallError(CallErrorKind::BindingError, bindings) + if bindings.is_single() => + { + reporter + .may_not( + "It may not have an `__iter__` method \ + and its `__getitem__` method has an incorrect signature \ + for the old-style iteration protocol", + ) + .info( + "`__getitem__` must be at least as permissive as \ + `def __getitem__(self, key: int): ...` \ + to satisfy the old-style iteration protocol", + ); + } + CallDunderError::CallError(CallErrorKind::BindingError, bindings) => { + reporter + .may_not(format_args!( + "It may not have an `__iter__` method \ + and its `__getitem__` method (with type `{dunder_getitem_type}`) \ + may have an incorrect signature for the old-style iteration protocol", + dunder_getitem_type = bindings.callable_type().display(db), + )) + .info( + "`__getitem__` must be at least as permissive as \ + `def __getitem__(self, key: int): ...` \ + to satisfy the old-style iteration protocol", + ); + } + }, + + Self::UnboundIterAndGetitemError { + dunder_getitem_error, + } => match dunder_getitem_error { + CallDunderError::MethodNotAvailable => { + reporter + .is_not("It doesn't have an `__iter__` method or a `__getitem__` method"); + } + CallDunderError::PossiblyUnbound(_) => { + reporter.is_not( + "It has no `__iter__` method and it may not have a `__getitem__` method", + ); + } + CallDunderError::CallError(CallErrorKind::NotCallable, bindings) => { + reporter.is_not(format_args!( + "It has no `__iter__` method and \ + its `__getitem__` attribute has type `{dunder_getitem_type}`, \ + which is not callable", + dunder_getitem_type = bindings.callable_type().display(db), + )); + } + CallDunderError::CallError(CallErrorKind::PossiblyNotCallable, bindings) + if bindings.is_single() => + { + reporter.may_not( + "It has no `__iter__` method and its `__getitem__` attribute \ + may not be callable", + ); + } + CallDunderError::CallError(CallErrorKind::PossiblyNotCallable, bindings) => { + reporter.may_not( + "It has no `__iter__` method and its `__getitem__` attribute is invalid", + ).info(format_args!( + "`__getitem__` has type `{dunder_getitem_type}`, which is not callable", + dunder_getitem_type = bindings.callable_type().display(db), + )); + } + CallDunderError::CallError(CallErrorKind::BindingError, bindings) + if bindings.is_single() => + { + reporter + .is_not( + "It has no `__iter__` method and \ + its `__getitem__` method has an incorrect signature \ + for the old-style iteration protocol", + ) + .info( + "`__getitem__` must be at least as permissive as \ + `def __getitem__(self, key: int): ...` \ + to satisfy the old-style iteration protocol", + ); + } + CallDunderError::CallError(CallErrorKind::BindingError, bindings) => { + reporter + .may_not(format_args!( + "It has no `__iter__` method and \ + its `__getitem__` method (with type `{dunder_getitem_type}`) \ + may have an incorrect signature for the old-style iteration protocol", + dunder_getitem_type = bindings.callable_type().display(db), + )) + .info( + "`__getitem__` must be at least as permissive as \ + `def __getitem__(self, key: int): ...` \ + to satisfy the old-style iteration protocol", + ); + } + }, + + IterationError::UnboundAiterError => { + reporter.is_not("It has no `__aiter__` method"); + } + } + } +} diff --git a/crates/ty_python_semantic/src/unpack.rs b/crates/ty_python_semantic/src/unpack.rs index cb07f2570a725..c9acc3fcfd95f 100644 --- a/crates/ty_python_semantic/src/unpack.rs +++ b/crates/ty_python_semantic/src/unpack.rs @@ -7,6 +7,7 @@ use crate::Db; use crate::ast_node_ref::AstNodeRef; use crate::semantic_index::expression::Expression; use crate::semantic_index::scope::{FileScopeId, ScopeId}; +use crate::types::EvaluationMode; /// This ingredient represents a single unpacking. /// @@ -102,26 +103,6 @@ impl<'db> UnpackValue<'db> { } } -#[derive(Clone, Copy, Debug, Hash, salsa::Update, get_size2::GetSize)] -pub(crate) enum EvaluationMode { - Sync, - Async, -} - -impl EvaluationMode { - pub(crate) const fn from_is_async(is_async: bool) -> Self { - if is_async { - EvaluationMode::Async - } else { - EvaluationMode::Sync - } - } - - pub(crate) const fn is_async(self) -> bool { - matches!(self, EvaluationMode::Async) - } -} - #[derive(Clone, Copy, Debug, Hash, salsa::Update, get_size2::GetSize)] pub(crate) enum UnpackKind { /// An iterable expression like the one in a `for` loop or a comprehension.