diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 14356813b7bb0..d1668fd50f445 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -3,6 +3,7 @@ use std::fmt::{self, Write}; use std::ops::{Bound, Deref}; use std::{cmp, iter}; +pub use coroutine::PackCoroutineLayout; use rustc_hashes::Hash64; use rustc_index::Idx; use rustc_index::bit_set::BitMatrix; @@ -210,17 +211,21 @@ impl LayoutCalculator { >( &self, local_layouts: &IndexSlice, - prefix_layouts: IndexVec, + relocated_upvars: &IndexSlice>, + upvar_layouts: IndexVec, variant_fields: &IndexSlice>, storage_conflicts: &BitMatrix, + pack: PackCoroutineLayout, tag_to_layout: impl Fn(Scalar) -> F, ) -> LayoutCalculatorResult { coroutine::layout( self, local_layouts, - prefix_layouts, + relocated_upvars, + upvar_layouts, variant_fields, storage_conflicts, + pack, tag_to_layout, ) } diff --git a/compiler/rustc_abi/src/layout/coroutine.rs b/compiler/rustc_abi/src/layout/coroutine.rs index 2b22276d4aed7..3b40ba57df734 100644 --- a/compiler/rustc_abi/src/layout/coroutine.rs +++ b/compiler/rustc_abi/src/layout/coroutine.rs @@ -21,6 +21,7 @@ use std::iter; +use rustc_data_structures::unord::UnordSet; use rustc_index::bit_set::{BitMatrix, DenseBitSet}; use rustc_index::{Idx, IndexSlice, IndexVec}; use tracing::{debug, trace}; @@ -30,6 +31,17 @@ use crate::{ StructKind, TagEncoding, Variants, WrappingRange, }; +/// This option controls how coroutine saved locals are packed +/// into the coroutine state data +#[derive(Debug, Clone, Copy)] +pub enum PackCoroutineLayout { + /// The classic layout where captures are always promoted to coroutine state prefix + Classic, + /// Captures are first saved into the `UNRESUME` state and promoted + /// when they are used across more than one suspension + CapturesOnly, +} + /// Overlap eligibility and variant assignment for each CoroutineSavedLocal. #[derive(Clone, Debug, PartialEq)] enum SavedLocalEligibility { @@ -74,6 +86,7 @@ fn coroutine_saved_local_eligibility( calc: &super::LayoutCalculator, local_layouts: &IndexSlice, - mut prefix_layouts: IndexVec, + relocated_upvars: &IndexSlice>, + upvar_layouts: IndexVec, variant_fields: &IndexSlice>, storage_conflicts: &BitMatrix, + pack: PackCoroutineLayout, tag_to_layout: impl Fn(Scalar) -> F, ) -> super::LayoutCalculatorResult { use SavedLocalEligibility::*; let (ineligible_locals, assignments) = coroutine_saved_local_eligibility(local_layouts.len(), variant_fields, storage_conflicts); + debug!(?ineligible_locals); - // Build a prefix layout, including "promoting" all ineligible - // locals as part of the prefix. We compute the layout of all of - // these fields at once to get optimal packing. - let tag_index = prefix_layouts.next_index(); + // Build a prefix layout, consisting of only the state tag and, as per request, upvars + let tag_index = match pack { + PackCoroutineLayout::CapturesOnly => FieldIdx::new(0), + PackCoroutineLayout::Classic => upvar_layouts.next_index(), + }; // `variant_fields` already accounts for the reserved variants, so no need to add them. let max_discr = (variant_fields.len() - 1) as u128; @@ -168,18 +187,38 @@ pub(super) fn layout< valid_range: WrappingRange { start: 0, end: max_discr }, }; - let promoted_layouts = ineligible_locals.iter().map(|local| local_layouts[local]); - prefix_layouts.push(tag_to_layout(tag)); - prefix_layouts.extend(promoted_layouts); + let upvars_in_unresumed: UnordSet<_> = + variant_fields[VariantIdx::new(0)].iter().copied().collect(); + let promoted_layouts = ineligible_locals.iter().filter_map(|local| { + if matches!(pack, PackCoroutineLayout::Classic) && upvars_in_unresumed.contains(&local) { + // We do not need to promote upvars, they are already in the upvar region + None + } else { + Some(local_layouts[local]) + } + }); + // FIXME: when we introduce more pack scheme, we need to change the prefix layout here + let prefix_layouts: IndexVec<_, _> = match pack { + PackCoroutineLayout::Classic => { + // Classic scheme packs the states as follows + // [ .. , , ] ++ + // In addition, UNRESUME overlaps with the part + upvar_layouts.into_iter().chain([tag_to_layout(tag)]).chain(promoted_layouts).collect() + } + PackCoroutineLayout::CapturesOnly => { + [tag_to_layout(tag)].into_iter().chain(promoted_layouts).collect() + } + }; + debug!(?prefix_layouts, ?pack); let prefix = calc.univariant(&prefix_layouts, &ReprOptions::default(), StructKind::AlwaysSized)?; let (prefix_size, prefix_align) = (prefix.size, prefix.align); - // Split the prefix layout into the "outer" fields (upvars and - // discriminant) and the "promoted" fields. Promoted fields will - // get included in each variant that requested them in - // CoroutineLayout. + // Split the prefix layout into the discriminant and + // the "promoted" fields. + // Promoted fields will get included in each variant + // that requested them in CoroutineLayout. debug!("prefix = {:#?}", prefix); let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields { FieldsShape::Arbitrary { mut offsets, memory_index } => { @@ -213,24 +252,75 @@ pub(super) fn layout< _ => unreachable!(), }; + // Here we start to compute layout of each state variant let mut size = prefix.size; let mut align = prefix.align; let variants = variant_fields .iter_enumerated() .map(|(index, variant_fields)| { + // Special case: UNRESUMED overlaps with the upvar region of the prefix, + // so that moving upvars may eventually become a no-op. + let is_unresumed = index.index() == 0; + if is_unresumed && matches!(pack, PackCoroutineLayout::Classic) { + let fields = FieldsShape::Arbitrary { + offsets: (0..tag_index.index()).map(|i| outer_fields.offset(i)).collect(), + memory_index: (0..tag_index.index()) + .map(|i| { + (outer_fields.memory_index(i) + promoted_memory_index.len()) as u32 + }) + .collect(), + }; + let align = prefix.align; + let size = prefix.size; + return Ok(LayoutData { + fields, + variants: Variants::Single { index }, + backend_repr: BackendRepr::Memory { sized: true }, + largest_niche: None, + uninhabited: false, + align, + size, + max_repr_align: None, + unadjusted_abi_align: align.abi, + randomization_seed: Default::default(), + }); + } + let mut is_ineligible = IndexVec::from_elem_n(None, variant_fields.len()); + for (field, &local) in variant_fields.iter_enumerated() { + if is_unresumed { + if let Some(inner_local) = relocated_upvars[local] + && let Ineligible(Some(promoted_field)) = assignments[inner_local] + { + is_ineligible.insert(field, promoted_field); + continue; + } + } + match assignments[local] { + Assigned(v) if v == index => {} + Ineligible(Some(promoted_field)) => { + is_ineligible.insert(field, promoted_field); + } + Ineligible(None) => { + panic!("an ineligible local should have been promoted into the prefix") + } + Assigned(_) => { + panic!("an eligible local should have been assigned to exactly one variant") + } + Unassigned => { + panic!("each saved local should have been inspected at least once") + } + } + } // Only include overlap-eligible fields when we compute our variant layout. - let variant_only_tys = variant_fields - .iter() - .filter(|local| match assignments[**local] { - Unassigned => unreachable!(), - Assigned(v) if v == index => true, - Assigned(_) => unreachable!("assignment does not match variant"), - Ineligible(_) => false, + let fields: IndexVec<_, _> = variant_fields + .iter_enumerated() + .filter_map(|(field, &local)| { + if is_ineligible.contains(field) { None } else { Some(local_layouts[local]) } }) - .map(|local| local_layouts[*local]); + .collect(); let mut variant = calc.univariant( - &variant_only_tys.collect::>(), + &fields, &ReprOptions::default(), StructKind::Prefixed(prefix_size, prefix_align.abi), )?; @@ -254,19 +344,14 @@ pub(super) fn layout< IndexVec::from_elem_n(FieldIdx::new(invalid_field_idx), invalid_field_idx); let mut offsets_and_memory_index = iter::zip(offsets, memory_index); - let combined_offsets = variant_fields + let combined_offsets = is_ineligible .iter_enumerated() - .map(|(i, local)| { - let (offset, memory_index) = match assignments[*local] { - Unassigned => unreachable!(), - Assigned(_) => { - let (offset, memory_index) = offsets_and_memory_index.next().unwrap(); - (offset, promoted_memory_index.len() as u32 + memory_index) - } - Ineligible(field_idx) => { - let field_idx = field_idx.unwrap(); - (promoted_offsets[field_idx], promoted_memory_index[field_idx]) - } + .map(|(i, &is_ineligible)| { + let (offset, memory_index) = if let Some(field_idx) = is_ineligible { + (promoted_offsets[field_idx], promoted_memory_index[field_idx]) + } else { + let (offset, memory_index) = offsets_and_memory_index.next().unwrap(); + (offset, promoted_memory_index.len() as u32 + memory_index) }; combined_inverse_memory_index[memory_index] = i; offset diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index de44c8755a078..fa9d947cd9f1d 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -67,7 +67,9 @@ pub use canon_abi::{ArmCall, CanonAbi, InterruptKind, X86Call}; pub use extern_abi::CVariadicStatus; pub use extern_abi::{ExternAbi, all_names}; #[cfg(feature = "nightly")] -pub use layout::{FIRST_VARIANT, FieldIdx, Layout, TyAbiInterface, TyAndLayout, VariantIdx}; +pub use layout::{ + FIRST_VARIANT, FieldIdx, Layout, PackCoroutineLayout, TyAbiInterface, TyAndLayout, VariantIdx, +}; pub use layout::{LayoutCalculator, LayoutCalculatorError}; /// Requirements for a `StableHashingContext` to be used in this crate. diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index a57689a45b67b..a8fab1be0eca2 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -2478,7 +2478,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { // If the local may have been initialized, and it is now currently being // mutated, then it is justified to be annotated with the `mut` // keyword, since the mutation may be a possible reassignment. - if is_local_mutation_allowed != LocalMutationIsAllowed::Yes + if !matches!(is_local_mutation_allowed, LocalMutationIsAllowed::Yes) && self.is_local_ever_initialized(local, state).is_some() { self.used_mut.insert(local); diff --git a/compiler/rustc_borrowck/src/path_utils.rs b/compiler/rustc_borrowck/src/path_utils.rs index 2c94a32d369ce..6b3169223029a 100644 --- a/compiler/rustc_borrowck/src/path_utils.rs +++ b/compiler/rustc_borrowck/src/path_utils.rs @@ -3,7 +3,7 @@ use std::ops::ControlFlow; use rustc_abi::FieldIdx; use rustc_data_structures::graph::dominators::Dominators; use rustc_middle::mir::{BasicBlock, Body, Location, Place, PlaceRef, ProjectionElem}; -use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::{CapturedPlace, TyCtxt}; use tracing::debug; use crate::borrow_set::{BorrowData, BorrowSet, TwoPhaseActivation}; @@ -133,7 +133,7 @@ pub(super) fn borrow_of_local_data(place: Place<'_>) -> bool { /// of a closure type. pub(crate) fn is_upvar_field_projection<'tcx>( tcx: TyCtxt<'tcx>, - upvars: &[&rustc_middle::ty::CapturedPlace<'tcx>], + upvars: &[&CapturedPlace<'tcx>], place_ref: PlaceRef<'tcx>, body: &Body<'tcx>, ) -> Option { diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index d1d0bff8fe06b..a6976e24af12c 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -25,8 +25,8 @@ use rustc_middle::traits::query::NoSolution; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::cast::CastTy; use rustc_middle::ty::{ - self, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, CoroutineArgsExt, - GenericArgsRef, Ty, TyCtxt, TypeVisitableExt, UserArgs, UserTypeAnnotationIndex, fold_regions, + self, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, GenericArgsRef, Ty, TyCtxt, + TypeVisitableExt, UserArgs, UserTypeAnnotationIndex, fold_regions, }; use rustc_middle::{bug, span_bug}; use rustc_mir_dataflow::move_paths::MoveData; @@ -2165,14 +2165,15 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { } } AggregateKind::Coroutine(_, args) => { - // It doesn't make sense to look at a field beyond the prefix; - // these require a variant index, and are not initialized in - // aggregate rvalues. - match args.as_coroutine().prefix_tys().get(field_index.as_usize()) { - Some(ty) => Ok(*ty), - None => Err(FieldAccessError::OutOfRange { - field_count: args.as_coroutine().prefix_tys().len(), - }), + // It doesn't make sense to look at a field beyond the captured + // upvars. + // Otherwise it require a variant index, and are not initialized + // in aggregate rvalues. + let upvar_tys = &args.as_coroutine().upvar_tys(); + if let Some(ty) = upvar_tys.get(field_index.as_usize()) { + Ok(*ty) + } else { + Err(FieldAccessError::OutOfRange { field_count: upvar_tys.len() }) } } AggregateKind::CoroutineClosure(_, args) => { diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 7d50548b40262..cbef1848fefec 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -891,6 +891,9 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: let variant_dest = lval.downcast_variant(fx, variant_index); (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_def_id, _args) => { + (FIRST_VARIANT, lval.downcast_variant(fx, FIRST_VARIANT), None) + } _ => (FIRST_VARIANT, lval, None), }; if active_field_index.is_some() { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 5535ebe65859e..2bd8633b63852 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -15,9 +15,7 @@ use rustc_middle::bug; use rustc_middle::ty::layout::{ HasTypingEnv, LayoutOf, TyAndLayout, WIDE_PTR_ADDR, WIDE_PTR_EXTRA, }; -use rustc_middle::ty::{ - self, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility, -}; +use rustc_middle::ty::{self, AdtKind, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility}; use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::{ DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, Span, Symbol, hygiene, @@ -1152,7 +1150,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( closure_or_coroutine_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() { - ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().prefix_tys()), + ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().upvar_tys()), ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), ty::CoroutineClosure(def_id, args) => (def_id, args.as_coroutine_closure().upvar_tys()), _ => { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 4ecc3086e1bdf..769c34f51a04c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -721,7 +721,6 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( let coroutine_layout = cx.tcx.coroutine_layout(coroutine_def_id, coroutine_args.args).unwrap(); - let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); let variant_range = coroutine_args.variant_range(coroutine_def_id, cx.tcx); let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); @@ -761,7 +760,6 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( coroutine_type_and_layout, coroutine_type_di_node, coroutine_layout, - common_upvar_names, ); let span = coroutine_layout.variant_source_info[variant_index].span; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index caff358607974..81425c71ac645 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -5,12 +5,11 @@ use rustc_codegen_ssa::debuginfo::type_names::{compute_debuginfo_type_name, cpp_ use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_hir::def::CtorKind; -use rustc_index::IndexSlice; use rustc_middle::bug; use rustc_middle::mir::CoroutineLayout; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef}; -use rustc_span::{Span, Symbol}; +use rustc_span::Span; use super::type_map::{DINodeCreationResult, UniqueTypeId}; use super::{SmallVec, size_and_align_of}; @@ -287,7 +286,6 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( coroutine_type_and_layout: TyAndLayout<'tcx>, coroutine_type_di_node: &'ll DIType, coroutine_layout: &CoroutineLayout<'tcx>, - common_upvar_names: &IndexSlice, ) -> &'ll DIType { let variant_name = CoroutineArgs::variant_name(variant_index); let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( @@ -298,11 +296,6 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( let variant_layout = coroutine_type_and_layout.for_variant(cx, variant_index); - let coroutine_args = match coroutine_type_and_layout.ty.kind() { - ty::Coroutine(_, args) => args.as_coroutine(), - _ => unreachable!(), - }; - type_map::build_type_with_children( cx, type_map::stub( @@ -317,7 +310,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( ), |cx, variant_struct_type_di_node| { // Fields that just belong to this variant/state - let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) + (0..variant_layout.fields.count()) .map(|field_index| { let coroutine_saved_local = coroutine_layout.variant_fields[variant_index] [FieldIdx::from_usize(field_index)]; @@ -340,29 +333,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( None, ) }) - .collect(); - - // Fields that are common to all states - let common_fields: SmallVec<_> = coroutine_args - .prefix_tys() - .iter() - .zip(common_upvar_names) - .enumerate() - .map(|(index, (upvar_ty, upvar_name))| { - build_field_di_node( - cx, - variant_struct_type_di_node, - upvar_name.as_str(), - cx.layout_of(upvar_ty), - coroutine_type_and_layout.fields.offset(index), - DIFlags::FlagZero, - type_di_node(cx, upvar_ty), - None, - ) - }) - .collect(); - - state_specific_fields.into_iter().chain(common_fields).collect() + .collect() }, |cx| build_generic_type_param_di_nodes(cx, coroutine_type_and_layout.ty), ) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 1ae6e6e5eecab..44f4426c699f9 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -186,9 +186,6 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( ) }; - let common_upvar_names = - cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); - // Build variant struct types let variant_struct_type_di_nodes: SmallVec<_> = variants .indices() @@ -216,7 +213,6 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( coroutine_type_and_layout, coroutine_type_di_node, coroutine_layout, - common_upvar_names, ), source_info, } diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 88a8e2a844cbc..b187cfcc14b70 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -934,13 +934,12 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue { } impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + #[instrument(level = "debug", skip(self, bx), ret)] fn maybe_codegen_consume_direct( &mut self, bx: &mut Bx, place_ref: mir::PlaceRef<'tcx>, ) -> Option> { - debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref); - match self.locals[place_ref.local] { LocalRef::Operand(mut o) => { // We only need to handle the projections that @@ -980,13 +979,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } + #[instrument(level = "debug", skip(self, bx), ret)] pub fn codegen_consume( &mut self, bx: &mut Bx, place_ref: mir::PlaceRef<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { - debug!("codegen_consume(place_ref={:?})", place_ref); - let ty = self.monomorphized_place_ty(place_ref); let layout = bx.cx().layout_of(ty); @@ -1005,13 +1003,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.load_operand(place) } + #[instrument(level = "debug", skip(self, bx), ret)] pub fn codegen_operand( &mut self, bx: &mut Bx, operand: &mir::Operand<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { - debug!("codegen_operand(operand={:?})", operand); - match *operand { mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => { self.codegen_consume(bx, place.as_ref()) diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 640f7211dc994..88c52b8643538 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -167,6 +167,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let variant_dest = dest.project_downcast(bx, variant_index); (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_, _) => { + (FIRST_VARIANT, dest.project_downcast(bx, FIRST_VARIANT), None) + } _ => (FIRST_VARIANT, dest, None), }; if active_field_index.is_some() { diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 1766bbe92480a..fe5e87bb8ab7d 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -301,6 +301,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let variant_dest = self.project_downcast(dest, variant_index)?; (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_def_id, _args) => { + (FIRST_VARIANT, self.project_downcast(dest, FIRST_VARIANT)?, None) + } mir::AggregateKind::RawPtr(..) => { // Pointers don't have "fields" in the normal sense, so the // projection-based code below would either fail in projection diff --git a/compiler/rustc_index/src/vec.rs b/compiler/rustc_index/src/vec.rs index 13f0dda180be9..90eb212e69bf5 100644 --- a/compiler/rustc_index/src/vec.rs +++ b/compiler/rustc_index/src/vec.rs @@ -197,6 +197,11 @@ impl IndexVec { pub fn append(&mut self, other: &mut Self) { self.raw.append(&mut other.raw); } + + #[inline] + pub fn debug_map_view(&self) -> IndexSliceMapView<'_, I, T> { + IndexSliceMapView(self.as_slice()) + } } /// `IndexVec` is often used as a map, so it provides some map-like APIs. @@ -220,14 +225,44 @@ impl IndexVec> { pub fn contains(&self, index: I) -> bool { self.get(index).and_then(Option::as_ref).is_some() } + + #[inline] + pub fn debug_map_view_compact(&self) -> IndexSliceMapViewCompact<'_, I, T> { + IndexSliceMapViewCompact(self.as_slice()) + } } +pub struct IndexSliceMapView<'a, I: Idx, T>(&'a IndexSlice); +pub struct IndexSliceMapViewCompact<'a, I: Idx, T>(&'a IndexSlice>); + impl fmt::Debug for IndexVec { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&self.raw, fmt) } } +impl<'a, I: Idx, T: fmt::Debug> fmt::Debug for IndexSliceMapView<'a, I, T> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut entries = fmt.debug_map(); + for (idx, val) in self.0.iter_enumerated() { + entries.entry(&idx, val); + } + entries.finish() + } +} + +impl<'a, I: Idx, T: fmt::Debug> fmt::Debug for IndexSliceMapViewCompact<'a, I, T> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut entries = fmt.debug_map(); + for (idx, val) in self.0.iter_enumerated() { + if let Some(val) = val { + entries.entry(&idx, val); + } + } + entries.finish() + } +} + impl Deref for IndexVec { type Target = IndexSlice; diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs index 2e6c9f207e26b..eeb396fe36d0d 100644 --- a/compiler/rustc_middle/src/mir/query.rs +++ b/compiler/rustc_middle/src/mir/query.rs @@ -9,7 +9,9 @@ use rustc_hir::def_id::LocalDefId; use rustc_index::IndexVec; use rustc_index::bit_set::BitMatrix; use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable}; +use rustc_session::config::PackCoroutineLayout; use rustc_span::{Span, Symbol}; +use rustc_type_ir::data_structures::IndexMap; use super::{ConstValue, SourceInfo}; use crate::ty::{self, CoroutineArgsExt, OpaqueHiddenType, Ty}; @@ -55,6 +57,29 @@ pub struct CoroutineLayout<'tcx> { #[type_foldable(identity)] #[type_visitable(ignore)] pub storage_conflicts: BitMatrix, + + /// This map `A -> B` allows later MIR passes, error reporters + /// and layout calculator to relate saved locals `A` sourced from upvars + /// and locals `B` that upvars are moved into. + /// + /// For instance, an upvar `_1.0` is assigned saved local `_s12`, + /// see notation of [`CoroutineSavedLocal`], in the UNRESUMED state and + /// further moved into the internal saved local `_s13`. + /// This map, therefore, establishes the mapping from `_s12` to `_s13`, + /// so that their memory layout within the coroutine should be overlapped. + #[type_foldable(identity)] + #[type_visitable(ignore)] + pub relocated_upvars: IndexMap, + + /// Coroutine layout packing + #[type_foldable(identity)] + #[type_visitable(ignore)] + pub pack: PackCoroutineLayout, +} + +impl<'tcx> CoroutineLayout<'tcx> { + /// The initial state of a coroutine + pub const UNRESUMED: VariantIdx = VariantIdx::ZERO; } impl Debug for CoroutineLayout<'_> { @@ -80,6 +105,7 @@ impl Debug for CoroutineLayout<'_> { map.finish() }) .field("storage_conflicts", &self.storage_conflicts) + .field("relocated_upvars", &self.relocated_upvars) .finish() } } diff --git a/compiler/rustc_middle/src/mir/statement.rs b/compiler/rustc_middle/src/mir/statement.rs index 3cbb4b70b062a..7d69a31fcf5ee 100644 --- a/compiler/rustc_middle/src/mir/statement.rs +++ b/compiler/rustc_middle/src/mir/statement.rs @@ -156,13 +156,13 @@ impl<'tcx> PlaceTy<'tcx> { .get(f.index()) .copied() .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")), - // Only prefix fields (upvars and current state) are - // accessible without a variant index. - ty::Coroutine(_, args) => { - args.as_coroutine().prefix_tys().get(f.index()).copied().unwrap_or_else(|| { - bug!("field {f:?} out of range of prefixes for {self_ty}") - }) - } + // Only upvars are accessible without a variant index. + ty::Coroutine(_, args) => args + .as_coroutine() + .upvar_tys() + .get(f.index()) + .copied() + .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")), ty::Tuple(tys) => tys .get(f.index()) .copied() diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index c27d47fcc0d8d..d0187befd0b73 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -409,15 +409,14 @@ impl<'tcx> fmt::Display for Instance<'tcx> { // async_drop_in_place::coroutine's poll function (through FutureDropPollShim proxy) fn resolve_async_drop_poll<'tcx>(mut cor_ty: Ty<'tcx>) -> Instance<'tcx> { let first_cor = cor_ty; - let ty::Coroutine(poll_def_id, proxy_args) = first_cor.kind() else { + let &ty::Coroutine(poll_def_id, proxy_args) = first_cor.kind() else { bug!(); }; - let poll_def_id = *poll_def_id; let mut child_ty = cor_ty; loop { - if let ty::Coroutine(child_def, child_args) = child_ty.kind() { + if let &ty::Coroutine(child_def, child_args) = child_ty.kind() { cor_ty = child_ty; - if *child_def == poll_def_id { + if child_def == poll_def_id { child_ty = child_args.first().unwrap().expect_ty(); continue; } else { diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index eefb913a33fa4..4072bb3342495 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -956,9 +956,10 @@ where ), Variants::Multiple { tag, tag_field, .. } => { if FieldIdx::from_usize(i) == tag_field { - return TyMaybeWithLayout::TyAndLayout(tag_layout(tag)); + TyMaybeWithLayout::TyAndLayout(tag_layout(tag)) + } else { + TyMaybeWithLayout::Ty(args.as_coroutine().upvar_tys()[i]) } - TyMaybeWithLayout::Ty(args.as_coroutine().prefix_tys()[i]) } }, diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 93d0c77c1daee..eeef59e3324a0 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -46,6 +46,7 @@ use rustc_macros::{ }; use rustc_query_system::ich::StableHashingContext; use rustc_serialize::{Decodable, Encodable}; +use rustc_session::config::PackCoroutineLayout; pub use rustc_session::lint::RegisteredTools; use rustc_span::hygiene::MacroKind; use rustc_span::{DUMMY_SP, ExpnId, ExpnKind, Ident, Span, Symbol, sym}; @@ -111,7 +112,7 @@ pub use self::typeck_results::{ use crate::error::{OpaqueHiddenTypeMismatch, TypeMismatchReason}; use crate::metadata::ModChild; use crate::middle::privacy::EffectiveVisibilities; -use crate::mir::{Body, CoroutineLayout, CoroutineSavedLocal, SourceInfo}; +use crate::mir::{Body, CoroutineLayout, CoroutineSavedLocal, CoroutineSavedTy, SourceInfo}; use crate::query::{IntoQueryParam, Providers}; use crate::ty; use crate::ty::codec::{TyDecoder, TyEncoder}; @@ -1872,39 +1873,68 @@ impl<'tcx> TyCtxt<'tcx> { .ok_or_else(|| self.layout_error(LayoutError::Unknown(ty()))) } + /// Construct the trivial coroutine layout for async_drop of a coroutine + /// + /// The only upvar in question is the coroutine state machine to be dropped. + #[instrument(level = "debug", ret, skip(self))] + fn build_async_drop_trivial_coroutine_layout( + self, + def_id: DefId, + upvar_ty: Ty<'tcx>, + ) -> CoroutineLayout<'tcx> { + let span = self.def_span(def_id); + let source_info = SourceInfo::outermost(span); + let mut field_tys = IndexVec::new(); + let field_names = [None].into(); + let upvar_saved_local = + field_tys.push(CoroutineSavedTy { ty: upvar_ty, source_info, ignore_for_traits: true }); + // Even minimal, the trivial coroutine has 3 states (RESERVED_VARIANTS), + // so variant_fields and variant_source_info should have 3 elements. + let mut variant_fields: IndexVec> = + iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + variant_fields[VariantIdx::ZERO].push(upvar_saved_local); + let variant_source_info: IndexVec = + iter::repeat(source_info).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + let mut storage_conflicts = BitMatrix::new(1, 1); + storage_conflicts.insert(upvar_saved_local, upvar_saved_local); + + let relocated_upvars = Default::default(); + CoroutineLayout { + field_tys, + field_names, + variant_fields, + variant_source_info, + storage_conflicts, + relocated_upvars, + pack: PackCoroutineLayout::CapturesOnly, + } + } + /// Returns layout of a coroutine. Layout might be unavailable if the /// coroutine is tainted by errors. + #[instrument(level = "debug", skip(self), ret)] pub fn coroutine_layout( self, def_id: DefId, args: GenericArgsRef<'tcx>, ) -> Result<&'tcx CoroutineLayout<'tcx>, &'tcx LayoutError<'tcx>> { if self.is_async_drop_in_place_coroutine(def_id) { - // layout of `async_drop_in_place::{closure}` in case, - // when T is a coroutine, contains this internal coroutine's ptr in upvars - // and doesn't require any locals. Here is an `empty coroutine's layout` + // Layout of `async_drop_in_place::{closure}` when T is a coroutine + // It contains exactly one upvar which is a raw pointer to this inner coroutine + // and it does not suspend. let arg_cor_ty = args.first().unwrap().expect_ty(); - if arg_cor_ty.is_coroutine() { - let span = self.def_span(def_id); - let source_info = SourceInfo::outermost(span); - // Even minimal, empty coroutine has 3 states (RESERVED_VARIANTS), - // so variant_fields and variant_source_info should have 3 elements. - let variant_fields: IndexVec> = - iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); - let variant_source_info: IndexVec = - iter::repeat(source_info).take(CoroutineArgs::RESERVED_VARIANTS).collect(); - let proxy_layout = CoroutineLayout { - field_tys: [].into(), - field_names: [].into(), - variant_fields, - variant_source_info, - storage_conflicts: BitMatrix::new(0, 0), - }; - return Ok(self.arena.alloc(proxy_layout)); + if let &ty::Coroutine(_did, _args) = arg_cor_ty.kind() { + debug!("it is a trivial coroutine"); + let upvar_ty = Ty::new_mut_ptr(self, arg_cor_ty); + Ok(self + .arena + .alloc(self.build_async_drop_trivial_coroutine_layout(def_id, upvar_ty))) } else { + debug!("it is an async drop coroutine"); self.async_drop_coroutine_layout(def_id, args) } } else { + debug!("it is an ordinary coroutine"); self.ordinary_coroutine_layout(def_id, args) } } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index ad8b1fd0693b0..a225140ad0466 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -150,13 +150,6 @@ impl<'tcx> ty::CoroutineArgs> { }) }) } - - /// This is the types of the fields of a coroutine which are not stored in a - /// variant. - #[inline] - fn prefix_tys(self) -> &'tcx List> { - self.upvar_tys() - } } #[derive(Debug, Copy, Clone, HashStable, TypeFoldable, TypeVisitable)] diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 2ccd8178e667b..0d344ef2d1d86 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -52,7 +52,9 @@ mod by_move_body; mod drop; -use std::{iter, ops}; +mod relocate_upvars; + +use std::ops::Deref; pub(super) use by_move_body::coroutine_by_move_body_def_id; use drop::{ @@ -60,14 +62,16 @@ use drop::{ create_coroutine_drop_shim_proxy_async, elaborate_coroutine_drops, expand_async_drops, has_expandable_async_drops, insert_clean_drop, }; +pub(super) use relocate_upvars::RelocateUpvars; use rustc_abi::{FieldIdx, VariantIdx}; use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::unord::UnordMap; use rustc_errors::pluralize; use rustc_hir as hir; use rustc_hir::lang_items::LangItem; use rustc_hir::{CoroutineDesugaring, CoroutineKind}; use rustc_index::bit_set::{BitMatrix, DenseBitSet, GrowableBitSet}; -use rustc_index::{Idx, IndexVec, indexvec}; +use rustc_index::{Idx, IndexSlice, IndexVec, indexvec}; use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::util::Discr; @@ -82,13 +86,15 @@ use rustc_mir_dataflow::impls::{ use rustc_mir_dataflow::{ Analysis, Results, ResultsCursor, ResultsVisitor, visit_reachable_results, }; +use rustc_session::config::PackCoroutineLayout; use rustc_span::def_id::{DefId, LocalDefId}; use rustc_span::source_map::dummy_spanned; use rustc_span::symbol::sym; -use rustc_span::{DUMMY_SP, Span}; +use rustc_span::{DUMMY_SP, Span, Symbol, kw}; use rustc_trait_selection::error_reporting::InferCtxtErrorExt; use rustc_trait_selection::infer::TyCtxtInferExt as _; use rustc_trait_selection::traits::{ObligationCause, ObligationCauseCode, ObligationCtxt}; +use smallvec::{SmallVec, smallvec}; use tracing::{debug, instrument, trace}; use crate::deref_separator::deref_finder; @@ -102,6 +108,8 @@ struct RenameLocalVisitor<'tcx> { tcx: TyCtxt<'tcx>, } +const VARIANT_UNRESUMED: VariantIdx = VariantIdx::from_usize(CoroutineArgs::UNRESUMED); + impl<'tcx> MutVisitor<'tcx> for RenameLocalVisitor<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx @@ -159,7 +167,7 @@ impl<'tcx> MutVisitor<'tcx> for SelfArgVisitor<'tcx> { } } -#[tracing::instrument(level = "trace", skip(tcx))] +#[instrument(level = "trace", skip(tcx))] fn replace_base<'tcx>(place: &mut Place<'tcx>, new_base: Place<'tcx>, tcx: TyCtxt<'tcx>) { place.local = new_base.local; @@ -167,7 +175,7 @@ fn replace_base<'tcx>(place: &mut Place<'tcx>, new_base: Place<'tcx>, tcx: TyCtx new_projection.append(&mut place.projection.to_vec()); place.projection = tcx.mk_place_elems(&new_projection); - tracing::trace!(?place); + trace!(?place); } const SELF_ARG: Local = Local::from_u32(1); @@ -201,7 +209,7 @@ struct TransformVisitor<'tcx> { storage_liveness: IndexVec>>, // A list of suspension points, generated during the transform - suspension_points: Vec>, + suspension_points: IndexVec>, // The set of locals that have no `StorageLive`/`StorageDead` annotations. always_live_locals: DenseBitSet, @@ -272,7 +280,7 @@ impl<'tcx> TransformVisitor<'tcx> { // `core::ops::CoroutineState` only has single element tuple variants, // so we can just write to the downcasted first field and then set the // discriminant to the appropriate variant. - #[tracing::instrument(level = "trace", skip(self, statements))] + #[instrument(level = "trace", skip(self, statements))] fn make_state( &self, val: Operand<'tcx>, @@ -347,7 +355,7 @@ impl<'tcx> TransformVisitor<'tcx> { } // Create a Place referencing a coroutine struct field - #[tracing::instrument(level = "trace", skip(self), ret)] + #[instrument(level = "trace", skip(self), ret)] fn make_field(&self, variant_index: VariantIdx, idx: FieldIdx, ty: Ty<'tcx>) -> Place<'tcx> { let self_place = Place::from(SELF_ARG); let base = self.tcx.mk_place_downcast_unnamed(self_place, variant_index); @@ -358,7 +366,7 @@ impl<'tcx> TransformVisitor<'tcx> { } // Create a statement which changes the discriminant - #[tracing::instrument(level = "trace", skip(self))] + #[instrument(level = "trace", skip(self))] fn set_discr(&self, state_disc: VariantIdx, source_info: SourceInfo) -> Statement<'tcx> { let self_place = Place::from(SELF_ARG); Statement::new( @@ -371,7 +379,7 @@ impl<'tcx> TransformVisitor<'tcx> { } // Create a statement which reads the discriminant into a temporary - #[tracing::instrument(level = "trace", skip(self, body))] + #[instrument(level = "trace", skip(self, body))] fn get_discr(&self, body: &mut Body<'tcx>) -> (Statement<'tcx>, Place<'tcx>) { let temp_decl = LocalDecl::new(self.discr_ty, body.span); let local_decls_len = body.local_decls.push(temp_decl); @@ -386,7 +394,7 @@ impl<'tcx> TransformVisitor<'tcx> { } /// Swaps all references of `old_local` and `new_local`. - #[tracing::instrument(level = "trace", skip(self, body))] + #[instrument(level = "trace", skip(self, body))] fn replace_local(&mut self, old_local: Local, new_local: Local, body: &mut Body<'tcx>) { body.local_decls.swap(old_local, new_local); @@ -405,20 +413,30 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { self.tcx } - #[tracing::instrument(level = "trace", skip(self), ret)] + #[instrument(level = "trace", skip(self), ret)] fn visit_local(&mut self, local: &mut Local, _: PlaceContext, _location: Location) { assert!(!self.remap.contains(*local)); } - #[tracing::instrument(level = "trace", skip(self), ret)] + #[instrument(level = "trace", skip(self), ret)] fn visit_place(&mut self, place: &mut Place<'tcx>, _: PlaceContext, _location: Location) { // Replace an Local in the remap with a coroutine struct access if let Some(&Some((ty, variant_index, idx))) = self.remap.get(place.local) { replace_base(place, self.make_field(variant_index, idx, ty), self.tcx); + } else if let Place { local: ty::CAPTURE_STRUCT_LOCAL, projection } = *place + && let [first @ ProjectionElem::Field(..), rest @ ..] = &**projection + { + let projections: Vec<_> = [ProjectionElem::Downcast(None, VARIANT_UNRESUMED), *first] + .into_iter() + .chain(rest.iter().copied()) + .collect(); + let new_place = + Place::from(ty::CAPTURE_STRUCT_LOCAL).project_deeper(&projections, self.tcx); + *place = new_place; } } - #[tracing::instrument(level = "trace", skip(self, stmt), ret)] + #[instrument(level = "trace", skip(self, stmt), ret)] fn visit_statement(&mut self, stmt: &mut Statement<'tcx>, location: Location) { // Remove StorageLive and StorageDead statements for remapped locals if let StatementKind::StorageLive(l) | StatementKind::StorageDead(l) = stmt.kind @@ -429,7 +447,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { self.super_statement(stmt, location); } - #[tracing::instrument(level = "trace", skip(self, term), ret)] + #[instrument(level = "trace", skip(self, term), ret)] fn visit_terminator(&mut self, term: &mut Terminator<'tcx>, location: Location) { if let TerminatorKind::Return = term.kind { // `visit_basic_block_data` introduces `Return` terminators which read `RETURN_PLACE`. @@ -439,7 +457,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { self.super_terminator(term, location); } - #[tracing::instrument(level = "trace", skip(self, data), ret)] + #[instrument(level = "trace", skip(self, data), ret)] fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) { match data.terminator().kind { TerminatorKind::Return => { @@ -511,20 +529,20 @@ fn make_aggregate_adt<'tcx>( Rvalue::Aggregate(Box::new(AggregateKind::Adt(def_id, variant_idx, args, None, None)), operands) } -#[tracing::instrument(level = "trace", skip(tcx, body))] +#[instrument(level = "trace", skip(tcx, body))] fn make_coroutine_state_argument_indirect<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let coroutine_ty = body.local_decls[SELF_ARG].ty; + let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let ref_coroutine_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, coroutine_ty); // Replace the by value coroutine argument - body.local_decls[SELF_ARG].ty = ref_coroutine_ty; + body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty = ref_coroutine_ty; // Add a deref to accesses of the coroutine state SelfArgVisitor::new(tcx, tcx.mk_place_deref(SELF_ARG.into())).visit_body(body); } -#[tracing::instrument(level = "trace", skip(tcx, body))] +#[instrument(level = "trace", skip(tcx, body))] fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let coroutine_ty = body.local_decls[SELF_ARG].ty; @@ -536,7 +554,7 @@ fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body let pin_ref_coroutine_ty = Ty::new_adt(tcx, pin_adt_ref, args); // Replace the by ref coroutine argument - body.local_decls[SELF_ARG].ty = pin_ref_coroutine_ty; + body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty = pin_ref_coroutine_ty; let unpinned_local = body.local_decls.push(LocalDecl::new(ref_coroutine_ty, body.span)); @@ -586,7 +604,7 @@ fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body /// The async lowering step and the type / lifetime inference / checking are /// still using the `ResumeTy` indirection for the time being, and that indirection /// is removed here. After this transform, the coroutine body only knows about `&mut Context<'_>`. -#[tracing::instrument(level = "trace", skip(tcx, body), ret)] +#[instrument(level = "trace", skip(tcx, body), ret)] fn transform_async_context<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> Ty<'tcx> { let context_mut_ref = Ty::new_task_context(tcx); @@ -640,7 +658,7 @@ fn eliminate_get_context_call<'tcx>(bb_data: &mut BasicBlockData<'tcx>) -> Local } #[cfg_attr(not(debug_assertions), allow(unused))] -#[tracing::instrument(level = "trace", skip(tcx, body), ret)] +#[instrument(level = "trace", skip(tcx, body), ret)] fn replace_resume_ty_local<'tcx>( tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, @@ -677,15 +695,17 @@ fn transform_gen_context<'tcx>(body: &mut Body<'tcx>) { body.arg_count = 1; } +#[derive(Debug)] struct LivenessInfo { /// Which locals are live across any suspension point. saved_locals: CoroutineSavedLocals, /// The set of saved locals live at each suspension point. - live_locals_at_suspension_points: Vec>, + live_locals_at_suspension_points: + IndexVec>, /// Parallel vec to the above with SourceInfo for each yield terminator. - source_info_at_suspension_points: Vec, + source_info_at_suspension_points: IndexVec, /// For every saved local, the set of other saved locals that are /// storage-live at the same time as this local. We cannot overlap locals in @@ -697,6 +717,11 @@ struct LivenessInfo { storage_liveness: IndexVec>>, } +rustc_index::newtype_index! { + #[debug_format = "suspend_{}"] + struct SuspensionPointIdx {} +} + /// Computes which locals have to be stored in the state-machine for the /// given coroutine. /// @@ -705,16 +730,19 @@ struct LivenessInfo { /// case none exist, the local is considered to be always live. /// - a local has to be stored if it is either directly used after the /// the suspend point, or if it is live and has been previously borrowed. -#[tracing::instrument(level = "trace", skip(tcx, body))] +/// - finally, upvars conflict each other in storage; these `nr_upvars` +/// upvars are assigned to the last "phantom" saved locals. +#[instrument(level = "trace", skip(tcx, body))] fn locals_live_across_suspend_points<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - always_live_locals: &DenseBitSet, + always_live_locals: DenseBitSet, + nr_upvars: usize, movable: bool, ) -> LivenessInfo { // Calculate when MIR locals have live storage. This gives us an upper bound of their // lifetimes. - let mut storage_live = MaybeStorageLive::new(std::borrow::Cow::Borrowed(always_live_locals)) + let mut storage_live = MaybeStorageLive::new(std::borrow::Cow::Borrowed(&always_live_locals)) .iterate_to_fixpoint(tcx, body, None) .into_results_cursor(body); @@ -747,11 +775,12 @@ fn locals_live_across_suspend_points<'tcx>( MaybeLiveLocals.iterate_to_fixpoint(tcx, body, Some("coroutine")).into_results_cursor(body); let mut storage_liveness_map = IndexVec::from_elem(None, &body.basic_blocks); - let mut live_locals_at_suspension_points = Vec::new(); - let mut source_info_at_suspension_points = Vec::new(); + let mut live_locals_at_suspension_points = IndexVec::::default(); + let mut source_info_at_suspension_points = IndexVec::default(); let mut live_locals_at_any_suspension_point = DenseBitSet::new_empty(body.local_decls.len()); - for (block, data) in body.basic_blocks.iter_enumerated() { + for &block in body.basic_blocks.reverse_postorder() { + let data = &body.basic_blocks[block]; if let TerminatorKind::Yield { .. } = data.terminator().kind { let loc = Location { block, statement_index: data.statements.len() }; @@ -785,7 +814,7 @@ fn locals_live_across_suspend_points<'tcx>( live_locals.intersect(requires_storage_cursor.get()); // The coroutine argument is ignored. - live_locals.remove(SELF_ARG); + live_locals.remove(ty::CAPTURE_STRUCT_LOCAL); debug!("loc = {:?}, live_locals = {:?}", loc, live_locals); @@ -815,6 +844,35 @@ fn locals_live_across_suspend_points<'tcx>( &mut requires_storage.analysis, &requires_storage.results, ); + let storage_conflicts = if nr_upvars > 0 { + let nr_true_saved_locals = saved_locals.count(); + let nr_total_saved_locals = nr_true_saved_locals + nr_upvars; + let mut enlarged_storage_conflicts = + BitMatrix::new(nr_total_saved_locals, nr_total_saved_locals); + let mut upvars = DenseBitSet::new_empty(nr_total_saved_locals); + let mut ineligibles = upvars.clone(); + for (saved_local, local) in saved_locals.iter_enumerated() { + if always_live_locals.contains(local) { + ineligibles.insert(saved_local); + } + } + upvars.union(&ineligibles); + for row in storage_conflicts.rows() { + for column in storage_conflicts.iter(row) { + enlarged_storage_conflicts.insert(row, column); + } + } + for upvar in nr_true_saved_locals..nr_total_saved_locals { + let upvar = CoroutineSavedLocal::from_usize(upvar); + enlarged_storage_conflicts.union_row_with(&upvars, upvar); + } + for ineligible in ineligibles.iter() { + enlarged_storage_conflicts.union_row_with(&upvars, ineligible); + } + enlarged_storage_conflicts + } else { + storage_conflicts + }; LivenessInfo { saved_locals, @@ -830,6 +888,7 @@ fn locals_live_across_suspend_points<'tcx>( /// `CoroutineSavedLocal` is indexed in terms of the elements in this set; /// i.e. `CoroutineSavedLocal::new(1)` corresponds to the second local /// included in this set. +#[derive(Debug)] struct CoroutineSavedLocals(DenseBitSet); impl CoroutineSavedLocals { @@ -862,7 +921,7 @@ impl CoroutineSavedLocals { } } -impl ops::Deref for CoroutineSavedLocals { +impl Deref for CoroutineSavedLocals { type Target = DenseBitSet; fn deref(&self) -> &Self::Target { @@ -981,10 +1040,20 @@ impl StorageConflictVisitor<'_, '_> { } } -#[tracing::instrument(level = "trace", skip(liveness, body))] +#[derive(Debug, Copy, Clone)] +struct UpvarInfo { + name: Symbol, + span: Span, +} + +#[instrument(level = "debug", skip(liveness, body))] fn compute_layout<'tcx>( liveness: LivenessInfo, body: &Body<'tcx>, + upvar_tys: &[Ty<'tcx>], + upvar_infos: &[UpvarInfo], + pack: PackCoroutineLayout, + local_upvar_map: &IndexSlice, ) -> ( IndexVec, VariantIdx, FieldIdx)>>, CoroutineLayout<'tcx>, @@ -998,9 +1067,14 @@ fn compute_layout<'tcx>( storage_liveness, } = liveness; + // We need to later establish the map between upvars in UNRESUMED and locals in other states. + let local_upvar_map: UnordMap<_, _> = + local_upvar_map.iter_enumerated().map(|(field, &local)| (local, field)).collect(); + // Gather live local types and their indices. let mut locals = IndexVec::::new(); let mut tys = IndexVec::::new(); + let mut saved_local_upvar_map = UnordMap::default(); for (saved_local, local) in saved_locals.iter_enumerated() { debug!("coroutine saved local {:?} => {:?}", saved_local, local); @@ -1028,7 +1102,24 @@ fn compute_layout<'tcx>( debug!(?decl); tys.push(decl); + + if let Some(&field) = local_upvar_map.get(&local) { + saved_local_upvar_map.insert(field, saved_local); + } } + // These are the "saved locals" sourced from the UNRESUMED state. + let upvar_saved_locals: IndexVec = upvar_tys + .iter() + .zip(upvar_infos) + .map(|(&ty, info)| { + tys.push(CoroutineSavedTy { + ty, + source_info: SourceInfo::outermost(info.span), + ignore_for_traits: false, + }) + }) + .collect(); + debug!(?upvar_saved_locals); // Leave empty variants for the UNRESUMED, RETURNED, and POISONED states. // In debuginfo, these will correspond to the beginning (UNRESUMED) or end @@ -1045,12 +1136,14 @@ fn compute_layout<'tcx>( // Build the coroutine variant field list. // Create a map from local indices to coroutine struct indices. + let variant_fields: [_; CoroutineArgs::RESERVED_VARIANTS] = + [upvar_saved_locals.clone(), IndexVec::new(), IndexVec::new()]; let mut variant_fields: IndexVec> = - iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + variant_fields.into_iter().collect(); let mut remap = IndexVec::from_elem_n(None, saved_locals.domain_size()); - for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter().enumerate() { + for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter_enumerated() { let variant_index = - VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx); + VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx.as_usize()); let mut fields = IndexVec::new(); for (idx, saved_local) in live_locals.iter().enumerate() { fields.push(saved_local); @@ -1066,25 +1159,39 @@ fn compute_layout<'tcx>( } debug!("coroutine variant_fields = {:?}", variant_fields); debug!("coroutine storage_conflicts = {:#?}", storage_conflicts); + debug!(remap = ?remap.debug_map_view_compact()); + debug!(locals = ?locals.debug_map_view()); let mut field_names = IndexVec::from_elem(None, &tys); for var in &body.var_debug_info { - let VarDebugInfoContents::Place(place) = &var.value else { continue }; - let Some(local) = place.as_local() else { continue }; - let Some(&Some((_, variant, field))) = remap.get(local) else { - continue; - }; - - let saved_local = variant_fields[variant][field]; - field_names.get_or_insert_with(saved_local, || var.name); + debug!(?var); + if let VarDebugInfoContents::Place(place) = &var.value + && let Some(local) = place.local_or_deref_local() + && let Some(&Some((_, variant, field))) = remap.get(local) + { + let saved_local = variant_fields[variant][field]; + field_names.get_or_insert_with(saved_local, || var.name); + } + } + for (capture, &saved_local) in upvar_infos.iter().zip(&upvar_saved_locals) { + field_names.get_or_insert_with(saved_local, || capture.name); } + debug!(field_names = ?field_names.debug_map_view()); + let relocated_upvars = upvar_saved_locals + .iter_enumerated() + .filter_map(|(field, &source)| { + saved_local_upvar_map.get(&field).map(|&dest| (source, dest)) + }) + .collect(); let layout = CoroutineLayout { field_tys: tys, field_names, variant_fields, variant_source_info, storage_conflicts, + relocated_upvars, + pack, }; debug!(?remap); debug!(?layout); @@ -1260,7 +1367,7 @@ fn generate_poison_block_and_redirect_unwinds_there<'tcx>( } } -#[tracing::instrument(level = "trace", skip(tcx, transform, body))] +#[instrument(level = "trace", skip(tcx, transform, body))] fn create_coroutine_resume_function<'tcx>( tcx: TyCtxt<'tcx>, transform: TransformVisitor<'tcx>, @@ -1354,7 +1461,7 @@ impl Operation { } } -#[tracing::instrument(level = "trace", skip(transform, body))] +#[instrument(level = "trace", skip(transform, body))] fn create_cases<'tcx>( body: &mut Body<'tcx>, transform: &TransformVisitor<'tcx>, @@ -1410,14 +1517,20 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>( def_id: LocalDefId, ) -> Option> { let (body, _) = tcx.mir_promoted(def_id); - let body = body.borrow(); - let body = &*body; + let body = &*body.borrow(); // The first argument is the coroutine type passed by value let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; - let movable = match *coroutine_ty.kind() { - ty::Coroutine(def_id, _) => tcx.coroutine_movability(def_id) == hir::Movability::Movable, + let (movable, upvar_tys, upvar_infos) = match *coroutine_ty.kind() { + ty::Coroutine(def_id, args) => ( + matches!(tcx.coroutine_movability(def_id), hir::Movability::Movable), + args.as_coroutine().upvar_tys(), + tcx.closure_captures(def_id.expect_local()) + .iter() + .map(|info| UpvarInfo { name: info.var_ident.name, span: info.var_ident.span }) + .collect::>(), + ), ty::Error(_) => return None, _ => span_bug!(body.span, "unexpected coroutine type {}", coroutine_ty), }; @@ -1425,13 +1538,23 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>( // The witness simply contains all locals live across suspend points. let always_live_locals = always_storage_live_locals(body); - let liveness_info = locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); + debug!(?always_live_locals); + let liveness_info = + locals_live_across_suspend_points(tcx, body, always_live_locals, upvar_tys.len(), movable); // Extract locals which are live across suspension point into `layout` // `remap` gives a mapping from local indices onto coroutine struct indices // `storage_liveness` tells us which locals have live storage at suspension points - let (_, coroutine_layout, _) = compute_layout(liveness_info, body); - + let (_, coroutine_layout, _) = compute_layout( + liveness_info, + body, + upvar_tys, + &upvar_infos, + // Layout packing only happens at state transformation + // and we do not reflect this in analysis + PackCoroutineLayout::No, + IndexSlice::empty(), + ); check_suspend_tys(tcx, &coroutine_layout, body); check_field_tys_sized(tcx, &coroutine_layout, def_id); @@ -1486,25 +1609,58 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { // This only applies to coroutines return; }; - tracing::trace!(def_id = ?body.source.def_id()); + trace!(def_id = ?body.source.def_id()); let old_ret_ty = body.return_ty(); assert!(body.coroutine_drop().is_none() && body.coroutine_drop_async().is_none()); - - if let Some(dumper) = MirDumper::new(tcx, "coroutine_before", body) { - dumper.dump_mir(body); - } - // The first argument is the coroutine type passed by value - let coroutine_ty = body.local_decls.raw[1].ty; + let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let coroutine_kind = body.coroutine_kind().unwrap(); // Get the discriminant type and args which typeck computed - let ty::Coroutine(_, args) = coroutine_ty.kind() else { + let &ty::Coroutine(def_id, args) = coroutine_ty.kind() else { tcx.dcx().span_bug(body.span, format!("unexpected coroutine type {coroutine_ty}")); }; - let discr_ty = args.as_coroutine().discr_ty(tcx); + let coroutine_args = args.as_coroutine(); + let discr_ty = coroutine_args.discr_ty(tcx); + let upvar_tys = coroutine_args.upvar_tys(); + let local_upvar_map = if !matches!( + tcx.sess.opts.unstable_opts.pack_coroutine_layout, + rustc_session::config::PackCoroutineLayout::No + ) { + RelocateUpvars.run(tcx, def_id, upvar_tys, body) + } else { + Default::default() + }; + + if let Some(dumper) = MirDumper::new(tcx, "coroutine_before", body) { + dumper.dump_mir(body); + } + + let upvar_infos = match body.source.instance { + ty::InstanceKind::AsyncDropGlue(..) => { + smallvec![UpvarInfo { name: kw::SelfLower, span: DUMMY_SP }] + } + ty::InstanceKind::Item(_) => tcx + .closure_captures(def_id.expect_local()) + .iter() + .map(|info| UpvarInfo { name: info.var_ident.name, span: info.var_ident.span }) + .collect::>(), + ty::InstanceKind::Intrinsic(..) + | ty::InstanceKind::VTableShim(..) + | ty::InstanceKind::ReifyShim(..) + | ty::InstanceKind::FnPtrShim(..) + | ty::InstanceKind::Virtual(..) + | ty::InstanceKind::ClosureOnceShim { .. } + | ty::InstanceKind::ConstructCoroutineInClosureShim { .. } + | ty::InstanceKind::ThreadLocalShim(..) + | ty::InstanceKind::FutureDropPollShim(..) + | ty::InstanceKind::DropGlue(..) + | ty::InstanceKind::CloneShim(..) + | ty::InstanceKind::AsyncDropGlueCtorShim(..) + | ty::InstanceKind::FnPtrAddrShim(..) => unreachable!(), + }; let new_ret_ty = match coroutine_kind { CoroutineKind::Desugared(CoroutineDesugaring::Async, _) => { @@ -1559,9 +1715,14 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { } let always_live_locals = always_storage_live_locals(body); - let movable = coroutine_kind.movability() == hir::Movability::Movable; - let liveness_info = - locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); + let movable = matches!(coroutine_kind.movability(), hir::Movability::Movable); + let liveness_info = locals_live_across_suspend_points( + tcx, + body, + always_live_locals.clone(), + upvar_tys.len(), + movable, + ); if tcx.sess.opts.unstable_opts.validate_mir { let mut vis = EnsureCoroutineFieldAssignmentsNeverAlias { @@ -1576,14 +1737,21 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { // Extract locals which are live across suspension point into `layout` // `remap` gives a mapping from local indices onto coroutine struct indices // `storage_liveness` tells us which locals have live storage at suspension points - let (remap, layout, storage_liveness) = compute_layout(liveness_info, body); + let (remap, layout, storage_liveness) = compute_layout( + liveness_info, + body, + upvar_tys, + &upvar_infos, + tcx.sess.opts.unstable_opts.pack_coroutine_layout, + &local_upvar_map, + ); let can_return = can_return(tcx, body, body.typing_env(tcx)); // We rename RETURN_PLACE which has type mir.return_ty to new_ret_local // RETURN_PLACE then is a fresh unused local with type ret_ty. let new_ret_local = body.local_decls.push(LocalDecl::new(new_ret_ty, body.span)); - tracing::trace!(?new_ret_local); + trace!(?new_ret_local); // Run the transformation which converts Places from Local to coroutine struct // accesses for locals in `remap`. @@ -1596,7 +1764,7 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { remap, storage_liveness, always_live_locals, - suspension_points: Vec::new(), + suspension_points: IndexVec::default(), discr_ty, new_ret_local, old_ret_ty, diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs index 951ff69c19e3e..06e0cfc08e57c 100644 --- a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs +++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs @@ -69,7 +69,6 @@ use rustc_abi::{FieldIdx, VariantIdx}; use rustc_data_structures::steal::Steal; -use rustc_data_structures::unord::UnordMap; use rustc_hir as hir; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LocalDefId}; @@ -78,8 +77,18 @@ use rustc_middle::bug; use rustc_middle::hir::place::{Projection, ProjectionKind}; use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::{self, MirDumper}; +use rustc_middle::ty::data_structures::IndexMap; use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt, TypeVisitableExt}; +struct CaptureInfo<'tcx> { + /// Field index of the capture in the parent coroutine structure + remapped_idx: FieldIdx, + /// Type of the capture in the parent coroutine structure + remapped_ty: Ty<'tcx>, + peel_deref: bool, + bridging_projections: Vec>, +} + pub(crate) fn coroutine_by_move_body_def_id<'tcx>( tcx: TyCtxt<'tcx>, coroutine_def_id: LocalDefId, @@ -126,23 +135,27 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( .tuple_fields() .len(); - let field_remapping: UnordMap<_, _> = ty::analyze_coroutine_closure_captures( + let field_remapping: IndexMap<_, _> = ty::analyze_coroutine_closure_captures( tcx.closure_captures(parent_def_id).iter().copied(), tcx.closure_captures(coroutine_def_id).iter().skip(num_args).copied(), |(parent_field_idx, parent_capture), (child_field_idx, child_capture)| { // Store this set of additional projections (fields and derefs). // We need to re-apply them later. - let mut child_precise_captures = - child_capture.place.projections[parent_capture.place.projections.len()..].to_vec(); + let child_precise_captures = child_capture.place.projections + [parent_capture.place.projections.len()..] + .iter() + .copied(); // If the parent capture is by-ref, then we need to apply an additional // deref before applying any further projections to this place. - if parent_capture.is_by_ref() { - child_precise_captures.insert( - 0, - Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref }, - ); - } + let bridging_projections = if parent_capture.is_by_ref() { + [Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref }] + .into_iter() + .chain(child_precise_captures) + .collect() + } else { + child_precise_captures.collect() + }; // If the child capture is by-ref, then we need to apply a "ref" // projection (i.e. `&`) at the end. But wait! We don't have that // as a projection kind. So instead, we can apply its dual and @@ -168,8 +181,8 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( // Finally, store the type of the parent's captured place. We need // this when building the field projection in the MIR body later on. - let mut parent_capture_ty = parent_capture.place.ty(); - parent_capture_ty = match parent_capture.info.capture_kind { + let parent_capture_ty = parent_capture.place.ty(); + let remapped_ty = match parent_capture.info.capture_kind { ty::UpvarCapture::ByValue | ty::UpvarCapture::ByUse => parent_capture_ty, ty::UpvarCapture::ByRef(kind) => Ty::new_ref( tcx, @@ -181,19 +194,19 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( Some(( FieldIdx::from_usize(child_field_idx + num_args), - ( - FieldIdx::from_usize(parent_field_idx + num_args), - parent_capture_ty, + CaptureInfo { + remapped_idx: FieldIdx::from_usize(parent_field_idx + num_args), + remapped_ty, peel_deref, - child_precise_captures, - ), + bridging_projections, + }, )) }, ) .flatten() .collect(); - if coroutine_kind == ty::ClosureKind::FnOnce { + if matches!(coroutine_kind, ty::ClosureKind::FnOnce) { assert_eq!(field_remapping.len(), tcx.closure_captures(parent_def_id).len()); // The by-move body is just the body :) return coroutine_def_id.to_def_id(); @@ -212,6 +225,9 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( ); let mut by_move_body = body.clone(); + if let Some(mir_dumper) = MirDumper::new(tcx, "built", &body) { + mir_dumper.dump_mir(&body); + } MakeByMoveBody { tcx, field_remapping, by_move_coroutine_ty }.visit_body(&mut by_move_body); // This path is unique since we're in a query so we'll only be called once with `parent_def_id` @@ -254,7 +270,7 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( struct MakeByMoveBody<'tcx> { tcx: TyCtxt<'tcx>, - field_remapping: UnordMap, bool, Vec>)>, + field_remapping: IndexMap>, by_move_coroutine_ty: Ty<'tcx>, } @@ -275,8 +291,12 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { if place.local == ty::CAPTURE_STRUCT_LOCAL && let Some((&mir::ProjectionElem::Field(idx, _), projection)) = place.projection.split_first() - && let Some(&(remapped_idx, remapped_ty, peel_deref, ref bridging_projections)) = - self.field_remapping.get(&idx) + && let Some(&CaptureInfo { + remapped_idx, + remapped_ty, + peel_deref, + ref bridging_projections, + }) = self.field_remapping.get(&idx) { // As noted before, if the parent closure captures a field by value, and // the child captures a field by ref, then for the by-move body we're @@ -353,7 +373,7 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { local: ty::CAPTURE_STRUCT_LOCAL, projection: [mir::ProjectionElem::Field(idx, _)], } = place.as_ref() - && let Some(&(_, _, true, _)) = self.field_remapping.get(&idx) + && let Some(CaptureInfo { peel_deref: true, .. }) = self.field_remapping.get(idx) { statement.kind = mir::StatementKind::Nop; } diff --git a/compiler/rustc_mir_transform/src/coroutine/drop.rs b/compiler/rustc_mir_transform/src/coroutine/drop.rs index 2699a051a8fea..2a122e989f5ba 100644 --- a/compiler/rustc_mir_transform/src/coroutine/drop.rs +++ b/compiler/rustc_mir_transform/src/coroutine/drop.rs @@ -233,8 +233,14 @@ pub(super) fn has_expandable_async_drops<'tcx>( if body[bb].is_cleanup { continue; } - let TerminatorKind::Drop { place, target: _, unwind: _, replace: _, drop: _, async_fut } = - body[bb].terminator().kind + let TerminatorKind::Drop { + place, + target: _, + unwind: _, + replace: _, + drop: _, + async_fut: Some(_), + } = body[bb].terminator().kind else { continue; }; @@ -242,12 +248,9 @@ pub(super) fn has_expandable_async_drops<'tcx>( if place_ty == coroutine_ty { continue; } - if async_fut.is_none() { - continue; - } return true; } - return false; + false } /// Expand Drop terminator for async drops into mainline poll-switch and dropline poll-switch @@ -513,7 +516,8 @@ pub(super) fn elaborate_coroutine_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body elaborate_drop( &mut elaborator, *source_info, - Place::from(SELF_ARG), + Place::from(SELF_ARG) + .project_deeper(&[ProjectionElem::Downcast(None, VARIANT_UNRESUMED)], tcx), (), *target, unwind, diff --git a/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs new file mode 100644 index 0000000000000..e99b746cbd80f --- /dev/null +++ b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs @@ -0,0 +1,436 @@ +//! MIR rewrite pass to relocate upvars into native locals in the coroutine body +//! +//! # Summary +//! The current contract of coroutine upvars is as follows. +//! Coroutines are constructed, initially in state UNRESUMED, by copying or moving +//! captures into the `struct`-fields, which are also called prefix fields, +//! taking necessary references as per capture specification. +//! +//! ```text +//! Low address High address +//! ┌─────────┬─────────┬─────┬─────────────────────┬───────────────────────────────────────────────────────┬──────────────┐ +//! │ │ │ │ │ │ │ +//! │ Upvar 1 │ Upvar 2 │ ... │ Coroutine State Tag │ Ineligibles, aka. saved locals alive across 2+ states │ Other states │ +//! │ │ │ │ │ │ │ +//! └─────────┴─────────┴─────┴─────────────────────┴───────────────────────────────────────────────────────┴──────────────┘ +//! ``` +//! +//! In case some upvars are large and short-lived, the classic layout scheme can be wasteful. +//! One way to reduce the memory footprint is to +//! +//! This pass performs the following transformations. +//! 1. It generates a fresh batch of locals for each captured upvars. +//! +//! For each upvar, whether used or not, a fresh local is created with the same type. +//! The types respect the nature of the captures, being by-ref, by-ref-mut or by-value. +//! This is reflected in the results in the upvar analysis conducted in the HIR type-checking phase. +//! +//! 2. It replaces the places pointing into those upvars with places pointing into those locals instead +//! +//! Each place that starts with access into the coroutine structure `_1` is replaced with the fresh local as +//! the base. +//! Suppose we are to lower this coroutine into the MIR. +//! ```ignore (illustrative) +//! let mut captured = None; +//! let _ = #[coroutine] || { +//! yield (); +//! if let Some(inner) = &mut captured { +//! *inner = 42i32; // (*) +//! } +//! }; +//! ``` +//! `captured` is the only capture, whose mutable borrow is formally allotted to the first field `_1.0: &mut i32`. +//! The highlighted line `(*)` should be lowered, roughly, into MIR `(*_1.0) = const 42i32;`. +//! Now, by application of this pass, we create a new local `_4: &mut i32` and we perform the following +//! code transformation. +//! +//! A new block is constructed to just perform the relocation of this mutable borrow. +//! This block is inserted to the very beginning of the coroutine body control flow, +//! so that this is executed before any proper coroutine code as it transits from `UNRESUME` state to +//! any other state. +//! This "prologue" will look like the following. +//! ```ignore (illustrative) +//! StorageLive(_5); +//! StorageLive(_4); +//! _5 = move (_1.0); +//! _4 = move (_5); +//! StorageDead(_5); +//! ``` +//! Note that we also create a trampoline local `_5` of the same type. +//! +//! ### Intricacy around the trampoline local +//! The reason that we need the trampolines is because state transformation and coroutine +//! layout calculation is not aware of potential storage conflict between captures as struct fields +//! and other saved locals. +//! The only guarantee that we can work with is one where any coroutine layout calculator respects +//! the storage conflict contracts between *MIR locals*. +//! It is known that calculators do not consider struct fields, where captures reside, as MIR locals. +//! This is the source of potential memory overlaps. +//! For instance, in a hypothetical situation, +//! - `_1.0` is relocated to `_4`; +//! - `_1.1` is relocated to `_6`; +//! - `_4` and `_6` remains live at one of the first suspension state; +//! - `_4` occupies the same offset of `_1.1` and `_6` occupies the same offset of `_1.0` +//! as decided by some layout calculator; +//! In this scenario, without trampolining, the relocations introduce undefined behaviour. +//! +//! As a proposal for a future design, it is best that coroutine captures receive their own +//! MIR locals, possibly in a form of "function arguments" like `_1` itself. +//! The trampolining transformation already attempts to restore the semantics of MIR locals to +//! these captures and promoting them to "arguments" would make MIR safer to handle. +//! +//! One should note that this phase assumes that the initial built MIR respects the nature of captures. +//! For instance, if the upvar `_1.4` is instead a by-ref-mut capture of a value of type `T`, +//! this phase assumes that all access correctly built as operating on the place `(*_1.4)`. +//! Based on the assumption, this phase replaces `_1.4` with a fresh local `_34: &mut T` and +//! the correctness is still upheld. +//! +//! 3. It assembles an prologue to replace the current entry block. +//! +//! This prologue block transfers every captured upvar into its corresponding fresh local, *via scratch locals*. +//! The upvars are first completely moved into the scratch locals in batch, and then moved into the destination +//! locals in batch. +//! The reason is that it is possible that coroutine layout may change and the source memory location of +//! an upvar may not necessarily be mapped exactly to the same place as in the `UNRESUMED` state. +//! This is very possible, because the coroutine layout scheme at this moment remains opaque, +//! other than the contract that a saved local has a stable internal offset throughout its liveness span. +//! +//! While the current coroutine layout ensures that the same saved local has stable offsets throughout its lifetime, +//! technically the upvar in `UNRESUMED` state and their fresh locals are different saved locals. +//! This scratch locals re-establish safety so that the correct data permutation can take place, +//! when a future coroutine layout calculator sees the permutation fit. + +use itertools::Itertools; +use rustc_abi::FieldIdx; +use rustc_hir::def_id::DefId; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::{IndexSlice, IndexVec}; +use rustc_middle::bug; +use rustc_middle::mir::visit::{MutVisitor, PlaceContext}; +use rustc_middle::mir::*; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_span::{DUMMY_SP, Ident, Span, Symbol}; +use smallvec::{SmallVec, smallvec}; +use tracing::{debug, instrument}; + +use crate::elaborate_drop::{Unwind, elaborate_drop}; +use crate::pass_manager::MirPass; +use crate::patch::MirPatch; +use crate::shim::DropShimElaborator; + +pub(crate) struct RelocateUpvars; + +struct UpvarSubstitution<'tcx> { + /// Newly minted local into which the upvar is moved + local: Local, + /// The temporary local that the prologue will permute the upvars with + reloc: Local, + /// Place into the capture structure where this upvar is found + upvar_place: Place<'tcx>, + /// The span of the captured upvar from the parent body + span: Span, + /// Name of the upvar + name: Symbol, +} + +struct SubstituteUpvarVisitor<'tcx, 'a> { + tcx: TyCtxt<'tcx>, + mappings: &'a IndexSlice>, +} + +impl<'tcx, 'a> MutVisitor<'tcx> for SubstituteUpvarVisitor<'tcx, 'a> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_place(&mut self, place: &mut Place<'tcx>, _context: PlaceContext, location: Location) { + if let Place { local: ty::CAPTURE_STRUCT_LOCAL, projection } = place + && let [ProjectionElem::Field(field_idx, _ty), rest @ ..] = &***projection + { + let Some(&UpvarSubstitution { local, .. }) = self.mappings.get(*field_idx) else { + bug!( + "SubstituteUpvar: found {field_idx:?} @ {location:?} but there is no upvar for it" + ) + }; + let new_place = Place::from(local).project_deeper(rest, self.tcx); + *place = new_place; + } + } + + fn visit_local(&mut self, local: &mut Local, context: PlaceContext, location: Location) { + if *local == ty::CAPTURE_STRUCT_LOCAL { + bug!("found a stray _1 at {location:?} in context {context:?}") + } + } +} + +#[derive(Debug, Clone, Copy)] +struct RelocationInfo { + ident: Option, + immutable: bool, + by_ref: bool, +} + +impl RelocateUpvars { + #[instrument(level = "debug", skip_all, fields(def_id = ?body.source))] + pub(crate) fn run<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + def_id: DefId, + upvar_tys: &ty::List>, + body: &mut Body<'tcx>, + ) -> IndexVec { + if upvar_tys.is_empty() { + debug!("no upvar, skipping"); + return Default::default(); + } + + let upvar_infos = match body.source.instance { + ty::InstanceKind::AsyncDropGlue(..) => { + smallvec![RelocationInfo { ident: None, immutable: true, by_ref: false }] + } + ty::InstanceKind::Item(_) => tcx + .closure_captures(def_id.expect_local()) + .iter() + .map(|info| RelocationInfo { + ident: Some(Ident::new(info.to_symbol(), info.var_ident.span)), + immutable: matches!(info.mutability, ty::Mutability::Not), + by_ref: matches!(info.info.capture_kind, ty::UpvarCapture::ByRef(..)), + }) + .collect::>(), + ty::InstanceKind::Intrinsic(..) + | ty::InstanceKind::VTableShim(..) + | ty::InstanceKind::ReifyShim(..) + | ty::InstanceKind::FnPtrShim(..) + | ty::InstanceKind::Virtual(..) + | ty::InstanceKind::ClosureOnceShim { .. } + | ty::InstanceKind::ConstructCoroutineInClosureShim { .. } + | ty::InstanceKind::ThreadLocalShim(..) + | ty::InstanceKind::FutureDropPollShim(..) + | ty::InstanceKind::DropGlue(..) + | ty::InstanceKind::CloneShim(..) + | ty::InstanceKind::AsyncDropGlueCtorShim(..) + | ty::InstanceKind::FnPtrAddrShim(..) => unreachable!(), + }; + + // HACK: in case `AddRetag` is already run, we have one `Retag` at the body entrance + // so we need to make sure that first `Retag` is run. + let retags_in_start_block = body.basic_blocks[START_BLOCK] + .statements + .iter() + .find_position(|stmt| !matches!(stmt.kind, StatementKind::Retag(_, _))) + .map(|(loc, _)| loc); + + if let Some(mir_dumper) = MirDumper::new(tcx, "RelocateUpvars", body) { + mir_dumper.set_disambiguator(&"before").dump_mir(body); + } + + let mut substitution_mapping = IndexVec::new(); + for (field_idx, (upvar_ty, &captured)) in upvar_tys.iter().zip(&upvar_infos).enumerate() { + let span = captured.ident.map_or(DUMMY_SP, |ident| ident.span); + let name = if let Some(ident) = captured.ident { + ident.name + } else { + Symbol::intern(&format!("_{}", field_idx)) + }; + + let immutable = if captured.immutable { Mutability::Not } else { Mutability::Mut }; + let mut local_decl = LocalDecl::new(upvar_ty, span); + **local_decl.local_info.as_mut().unwrap_crate_local() = + LocalInfo::User(BindingForm::Var(VarBindingForm { + binding_mode: rustc_ast::BindingMode( + if captured.by_ref { + rustc_ast::ByRef::Yes(immutable) + } else { + rustc_ast::ByRef::No + }, + immutable, + ), + opt_ty_info: None, + opt_match_place: None, + pat_span: span, + introductions: vec![], + })); + let local = body.local_decls.push(local_decl); + + local_decl = LocalDecl::new(upvar_ty, span); + **local_decl.local_info.as_mut().unwrap_crate_local() = LocalInfo::Boring; + let reloc = body.local_decls.push(local_decl); + + let field_idx = FieldIdx::from_usize(field_idx); + let upvar_place = + tcx.mk_place_field(Place::from(ty::CAPTURE_STRUCT_LOCAL), field_idx, upvar_ty); + + substitution_mapping.push(UpvarSubstitution { local, reloc, upvar_place, span, name }); + } + rewrite_drop_coroutine_struct(tcx, body); + let local_upvar_map = substitution_mapping.iter().map(|sub| sub.local).collect(); + SubstituteUpvarVisitor { tcx, mappings: &substitution_mapping }.visit_body(body); + + insert_substitution_prologue(body, retags_in_start_block, &substitution_mapping); + patch_missing_storage_deads(body, &substitution_mapping); + hydrate_var_debug_info(body, &substitution_mapping); + if let Some(mir_dumper) = MirDumper::new(tcx, "RelocateUpvars", body) { + mir_dumper.set_disambiguator(&"after").dump_mir(body); + } + local_upvar_map + } +} + +impl<'tcx> MirPass<'tcx> for RelocateUpvars { + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let coroutine_ty = if body.yield_ty().is_some() + && let Some(decl) = body.local_decls.get(ty::CAPTURE_STRUCT_LOCAL) + { + decl.ty + } else { + // It fails the litmus test as a coroutine + bug!("RelocateUpvars can only be run on coroutines"); + }; + let (def_id, upvar_tys) = if let ty::Coroutine(def_id, args) = *coroutine_ty.kind() { + let args = args.as_coroutine(); + (def_id, args.upvar_tys()) + } else { + bug!("RelocateUpvars can only be run on coroutines"); + }; + self.run(tcx, def_id, upvar_tys, body); + } + + fn is_required(&self) -> bool { + true + } +} + +fn rewrite_drop_coroutine_struct<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + // Note that `elaborate_drops` only drops the upvars of a coroutine, and + // this is ok because `open_drop` can only be reached within that own + // coroutine's resume function. + let typing_env = body.typing_env(tcx); + + let mut elaborator = DropShimElaborator { + body, + patch: MirPatch::new(body), + tcx, + typing_env, + produce_async_drops: false, + }; + + for (block, block_data) in body.basic_blocks.iter_enumerated() { + let (target, unwind, source_info, dropline) = match block_data.terminator() { + Terminator { + source_info, + kind: TerminatorKind::Drop { place, target, unwind, replace: _, drop, async_fut: _ }, + } => { + if matches!(place.as_local(), Some(ty::CAPTURE_STRUCT_LOCAL)) { + (target, unwind, source_info, *drop) + } else { + continue; + } + } + _ => continue, + }; + let unwind = if block_data.is_cleanup { + Unwind::InCleanup + } else { + Unwind::To(match *unwind { + UnwindAction::Cleanup(tgt) => tgt, + UnwindAction::Continue => elaborator.patch.resume_block(), + UnwindAction::Unreachable => elaborator.patch.unreachable_cleanup_block(), + UnwindAction::Terminate(reason) => elaborator.patch.terminate_block(reason), + }) + }; + elaborate_drop( + &mut elaborator, + *source_info, + Place::from(ty::CAPTURE_STRUCT_LOCAL), + (), + *target, + unwind, + block, + dropline, + ); + } + elaborator.patch.apply(body); +} + +fn insert_substitution_prologue<'tcx>( + body: &mut Body<'tcx>, + retags_in_start_block: Option, + substitution_mapping: &IndexSlice>, +) { + // NOTE: START_BLOCK cannot have incoming edges + let mut stmts = Vec::with_capacity(5 * substitution_mapping.len()); + for &UpvarSubstitution { local, reloc, upvar_place, span, name: _ } in substitution_mapping { + // For each upvar-local _$i + let source_info = SourceInfo::outermost(span); + // StorageLive(_$i) + stmts.push(Statement::new(source_info, StatementKind::StorageLive(local))); + // Use a fresh local _$i' here, so as to avoid potential field permutation + // StorageLive(_$i') + stmts.push(Statement::new(source_info, StatementKind::StorageLive(reloc))); + // _$i' = move $ + stmts.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + reloc.into(), + Rvalue::Use(Operand::Move(upvar_place)), + ))), + )); + } + for &UpvarSubstitution { local, reloc, upvar_place: _, span, name: _ } in substitution_mapping { + let source_info = SourceInfo::outermost(span); + // _$i = move $i' + stmts.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + local.into(), + Rvalue::Use(Operand::Move(reloc.into())), + ))), + )); + stmts.push(Statement::new(source_info, StatementKind::StorageDead(reloc))); + } + + let start_stmts = &mut body.basic_blocks.as_mut_preserves_cfg()[START_BLOCK].statements; + let tail_stmts: Vec<_> = + start_stmts.splice(retags_in_start_block.unwrap_or(0).., stmts).collect(); + start_stmts.extend(tail_stmts); +} + +/// We need to mark relocated upvars daed for correctness, +/// as previously just the entire capture structure was marked dead and +/// now we need to mark them dead individually. +fn patch_missing_storage_deads<'tcx>( + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + let mut upvar_locals = DenseBitSet::new_empty(body.local_decls.len()); + for subst in substitution_mapping { + upvar_locals.insert(subst.local); + } + let basic_blocks = body.basic_blocks.as_mut_preserves_cfg(); + for data in basic_blocks.iter_mut() { + if !data.is_cleanup && matches!(data.terminator().kind, TerminatorKind::Return) { + for local in upvar_locals.iter() { + data.statements.push(Statement::new( + data.terminator().source_info, + StatementKind::StorageDead(local), + )); + } + } + } +} + +fn hydrate_var_debug_info<'tcx>( + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + for subst in substitution_mapping { + body.var_debug_info.push(VarDebugInfo { + name: subst.name, + source_info: SourceInfo::outermost(subst.span), + composite: None, + value: VarDebugInfoContents::Place(Place::from(subst.local)), + argument_index: None, + }); + } +} diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 4625b20fd8900..b2f002f773e3b 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -128,7 +128,7 @@ declare_passes! { pub mod cleanup_post_borrowck : CleanupPostBorrowck; mod copy_prop : CopyProp; - mod coroutine : StateTransform; + mod coroutine : RelocateUpvars, StateTransform; mod coverage : InstrumentCoverage; mod ctfe_limit : CtfeLimit; mod dataflow_const_prop : DataflowConstProp; diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index ab09cdf787ee7..987510f714b53 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -311,12 +311,21 @@ fn run_passes_inner<'tcx>( if let Some(dumper) = dumper { dumper.set_disambiguator(&"after").dump_mir(body); } + let (dialect, phase_num) = body.phase.index(); if validate { - validate_body(tcx, body, format!("after pass {pass_name}")); + validate_body( + tcx, + body, + format!("after pass {pass_name} {dialect}-{phase_num}-{:03}", body.pass_count), + ); } if lint { - lint_body(tcx, body, format!("after pass {pass_name}")); + lint_body( + tcx, + body, + format!("after pass {pass_name} {dialect}-{phase_num}-{:03}", body.pass_count), + ); } body.pass_count += 1; diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 85e340c0a02ab..14075d362ef2d 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -49,7 +49,7 @@ impl<'tcx> MutVisitor<'tcx> for FixProxyFutureDropVisitor<'tcx> { _context: PlaceContext, _location: Location, ) { - if place.local == Local::from_u32(1) { + if place.local == ty::CAPTURE_STRUCT_LOCAL { if place.projection.len() == 1 { assert!(matches!( place.projection.first(), @@ -66,9 +66,8 @@ impl<'tcx> MutVisitor<'tcx> for FixProxyFutureDropVisitor<'tcx> { } } +#[instrument(level = "debug", skip(tcx))] fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<'tcx> { - debug!("make_shim({:?})", instance); - let mut result = match instance { ty::InstanceKind::Item(..) => bug!("item {:?} passed to make_shim", instance), ty::InstanceKind::VTableShim(def_id) => { @@ -195,6 +194,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body< tcx, &mut body, &[ + &crate::coroutine::RelocateUpvars, // We must always relocate to ensure valid MIR &mentioned_items::MentionedItems, &abort_unwinding_calls::AbortUnwindingCalls, &add_call_guards::CriticalCallEdges, diff --git a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs index a0f1260cd986d..a3b718b164086 100644 --- a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs +++ b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs @@ -11,6 +11,7 @@ use rustc_middle::ty::{self, EarlyBinder, Ty, TyCtxt, TypeVisitableExt}; use super::*; use crate::patch::MirPatch; +#[instrument(level = "debug", skip(tcx))] pub(super) fn build_async_destructor_ctor_shim<'tcx>( tcx: TyCtxt<'tcx>, def_id: DefId, @@ -39,12 +40,12 @@ pub(super) fn build_async_destructor_ctor_shim<'tcx>( } // build_drop_shim analog for async drop glue (for generated coroutine poll function) +#[instrument(level = "debug", skip(tcx))] pub(super) fn build_async_drop_shim<'tcx>( tcx: TyCtxt<'tcx>, def_id: DefId, ty: Ty<'tcx>, ) -> Body<'tcx> { - debug!("build_async_drop_shim(def_id={:?}, ty={:?})", def_id, ty); let ty::Coroutine(_, parent_args) = ty.kind() else { bug!(); }; @@ -193,6 +194,7 @@ pub(super) fn build_future_drop_poll_shim<'tcx>( // `async_drop_in_place::{closure}.poll()` is converted into `T.future_drop_poll()`. // Every coroutine has its `poll` (calculate yourself a little further) // and its `future_drop_poll` (drop yourself a little further). +#[instrument(level = "debug", skip_all, fields(span))] fn build_adrop_for_coroutine_shim<'tcx>( tcx: TyCtxt<'tcx>, proxy_ty: Ty<'tcx>, @@ -200,33 +202,43 @@ fn build_adrop_for_coroutine_shim<'tcx>( span: Span, instance: ty::InstanceKind<'tcx>, ) -> Body<'tcx> { - let ty::Coroutine(coroutine_def_id, impl_args) = impl_ty.kind() else { + debug!("proxy_ty={proxy_ty:#?}"); + debug!("impl_ty={impl_ty:#?}"); + debug!("instance={instance:#?}"); + let &ty::Coroutine(coroutine_def_id, impl_args) = impl_ty.kind() else { bug!("build_adrop_for_coroutine_shim not for coroutine impl type: ({:?})", instance); }; let proxy_ref = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, proxy_ty); - // taking _1.0 (impl from Pin) - let pin_proxy_layout_local = Local::new(1); - let source_info = SourceInfo::outermost(span); - // converting `(_1: Pin<&mut CorLayout>, _2: &mut Context<'_>) -> Poll<()>` - // into `(_1: Pin<&mut ProxyLayout>, _2: &mut Context<'_>) -> Poll<()>` - // let mut _x: &mut CorLayout = &*_1.0.0; - // Replace old _1.0 accesses into _x accesses; - let body = tcx.optimized_mir(*coroutine_def_id).future_drop_poll().unwrap(); + let body = tcx.optimized_mir(coroutine_def_id).future_drop_poll().unwrap(); let mut body: Body<'tcx> = EarlyBinder::bind(body.clone()).instantiate(tcx, impl_args); + if let Some(mir_dumper) = MirDumper::new(tcx, "build_adrop_for_coroutine_shim_before", &body) { + mir_dumper.set_disambiguator(&0).dump_mir(&body); + } + debug!(?body.source.instance, "before"); body.source.instance = instance; body.phase = MirPhase::Runtime(RuntimePhase::Initial); body.var_debug_info.clear(); + // Here we convert `(_1: Pin<&mut InnerCoroutine>, _2: &mut Context<'_>) -> Poll<()>` + // into `(_1: Pin<&mut ProxyCoroutine>, _2: &mut Context<'_>) -> Poll<()>`. + // **The Schematic** + // We make a new local `_x`: + // let mut _x: &mut InnerCoroutine = &* ((*_1.0) as variant#0).0; + // Then replace projections on `_1.0` into projections on `_x`. + let pin_proxy_layout_local = Local::new(1); + let source_info = SourceInfo::outermost(span); let pin_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, span)); let args = tcx.mk_args(&[proxy_ref.into()]); let pin_proxy_ref = Ty::new_adt(tcx, pin_adt_ref, args); + // This is the type of the vanilla `InnerCoroutine` let cor_ref = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, impl_ty); let proxy_ref_local = body.local_decls.push(LocalDecl::new(proxy_ref, span)); let cor_ref_local = body.local_decls.push(LocalDecl::new(cor_ref, span)); + // Batch replacement of `_1.0` with `_x` FixProxyFutureDropVisitor { tcx, replace_to: cor_ref_local }.visit_body(&mut body); - // Now changing first arg from Pin<&mut ImplCoroutine> to Pin<&mut ProxyCoroutine> + // Now changing first arg from Pin<&mut InnerCoroutine> to Pin<&mut ProxyCoroutine> body.local_decls[pin_proxy_layout_local] = LocalDecl::new(pin_proxy_ref, span); { @@ -250,11 +262,15 @@ fn build_adrop_for_coroutine_shim<'tcx>( if ty != proxy_ty { let ty_ptr = Ty::new_mut_ptr(tcx, ty); let impl_ptr_place = Place::from(cor_ptr_local).project_deeper( - &[PlaceElem::Deref, PlaceElem::Field(FieldIdx::ZERO, ty_ptr)], + &[ + PlaceElem::Deref, + PlaceElem::Downcast(None, VariantIdx::ZERO), + PlaceElem::Field(FieldIdx::ZERO, ty_ptr), + ], tcx, ); cor_ptr_local = body.local_decls.push(LocalDecl::new(ty_ptr, span)); - // _cor_ptr = _proxy.0.0 (... .0) + // _cor_ptr = ((*_proxy.0) as variant#0).0 (... .0) body.basic_blocks_mut()[START_BLOCK].statements.insert( idx, Statement::new( @@ -283,6 +299,9 @@ fn build_adrop_for_coroutine_shim<'tcx>( ), ); } + if let Some(mir_dumper) = MirDumper::new(tcx, "build_adrop_for_coroutine_shim", &body) { + mir_dumper.set_disambiguator(&0).dump_mir(&body); + } body } @@ -342,10 +361,16 @@ fn build_adrop_for_adrop_shim<'tcx>( proxy_ty.find_async_drop_impl_coroutine(tcx, |ty| { if ty != proxy_ty { let ty_ptr = Ty::new_mut_ptr(tcx, ty); - let impl_ptr_place = Place::from(cor_ptr_local) - .project_deeper(&[PlaceElem::Deref, PlaceElem::Field(FieldIdx::ZERO, ty_ptr)], tcx); + let impl_ptr_place = Place::from(cor_ptr_local).project_deeper( + &[ + PlaceElem::Deref, + PlaceElem::Downcast(None, VariantIdx::ZERO), + PlaceElem::Field(FieldIdx::ZERO, ty_ptr), + ], + tcx, + ); cor_ptr_local = locals.push(LocalDecl::new(ty_ptr, span)); - // _cor_ptr = _proxy.0.0 (... .0) + // _cor_ptr = ((*_proxy.0) as variant#0).0 (... .0) statements.push(Statement::new( source_info, StatementKind::Assign(Box::new(( @@ -421,5 +446,8 @@ fn build_adrop_for_adrop_shim<'tcx>( let source = MirSource::from_instance(instance); let mut body = new_body(source, blocks, locals, sig.inputs().len(), span); body.phase = MirPhase::Runtime(RuntimePhase::Initial); - return body; + if let Some(mir_dumper) = MirDumper::new(tcx, "build_adrop_for_adrop_shim", &body) { + mir_dumper.set_disambiguator(&0).dump_mir(&body); + } + body } diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index da31600e8324c..f34ad2e5e0b2b 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -451,7 +451,7 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyLocals { // Only bother running the `LocalUpdater` if we actually found locals to remove. if map.iter().any(Option::is_none) { // Update references to all vars and tmps now - let mut updater = LocalUpdater { map, tcx }; + let mut updater = LocalUpdater { map: &map, tcx }; updater.visit_body_preserves_cfg(body); body.local_decls.shrink_to_fit(); @@ -647,12 +647,12 @@ fn remove_unused_definitions_helper(used_locals: &mut UsedLocals, body: &mut Bod } } -struct LocalUpdater<'tcx> { - map: IndexVec>, +struct LocalUpdater<'tcx, 'a> { + map: &'a IndexSlice>, tcx: TyCtxt<'tcx>, } -impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx> { +impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx, '_> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 5a9018a62c574..c055b31bb14b7 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -14,7 +14,7 @@ use rustc_middle::mir::*; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{ - self, CoroutineArgsExt, InstanceKind, ScalarInt, Ty, TyCtxt, TypeVisitableExt, Upcast, Variance, + self, InstanceKind, ScalarInt, Ty, TyCtxt, TypeVisitableExt, Upcast, Variance, }; use rustc_middle::{bug, span_bug}; use rustc_mir_dataflow::debuginfo::debuginfo_locals; @@ -786,14 +786,11 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { }; ty::EarlyBinder::bind(f_ty.ty).instantiate(self.tcx, args) - } else { - let Some(&f_ty) = args.as_coroutine().prefix_tys().get(f.index()) - else { - fail_out_of_bounds(self, location); - return; - }; - + } else if let Some(&f_ty) = args.as_coroutine().upvar_tys().get(f.index()) { f_ty + } else { + fail_out_of_bounds(self, location); + return; }; check_equal(self, location, f_ty); diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 8ff6d567422b9..b3fe99cd76f46 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -3252,9 +3252,9 @@ pub(crate) mod dep_tracking { CrateType, DebugInfo, DebugInfoCompression, ErrorOutputType, FmtDebug, FunctionReturn, InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail, LtoCli, MirStripDebugInfo, NextSolverConfig, Offload, OomStrategy, OptLevel, OutFileName, - OutputType, OutputTypes, PatchableFunctionEntry, Polonius, RemapPathScopeComponents, - ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, - SymbolManglingVersion, WasiExecModel, + OutputType, OutputTypes, PackCoroutineLayout, PatchableFunctionEntry, Polonius, + RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, + SwitchWithOptPath, SymbolManglingVersion, WasiExecModel, }; use crate::lint; use crate::utils::NativeLib; @@ -3357,6 +3357,7 @@ pub(crate) mod dep_tracking { Polonius, InliningThreshold, FunctionReturn, + PackCoroutineLayout, Align, ); @@ -3611,6 +3612,20 @@ pub enum FunctionReturn { ThunkExtern, } +/// Layout optimisation for Coroutines +#[derive(Clone, Copy, PartialEq, Eq, Hash, HashStable_Generic, Debug, Default)] +#[derive(Decodable, Encodable)] +pub enum PackCoroutineLayout { + /// Keep coroutine captured variables throughout all states + #[default] + No, + + /// Allow coroutine captured variables that are used only once + /// before the first suspension to be freed up for storage + /// in all other suspension states + CapturesOnly, +} + /// Whether extra span comments are included when dumping MIR, via the `-Z mir-include-spans` flag. /// By default, only enabled in the NLL MIR dumps, and disabled in all other passes. #[derive(Clone, Copy, Default, PartialEq, Debug)] diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 6dd90546de1b0..63c2604f341ee 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -805,6 +805,7 @@ mod desc { pub(crate) const parse_panic_strategy: &str = "either `unwind`, `abort`, or `immediate-abort`"; pub(crate) const parse_on_broken_pipe: &str = "either `kill`, `error`, or `inherit`"; pub(crate) const parse_patchable_function_entry: &str = "either two comma separated integers (total_nops,prefix_nops), with prefix_nops <= total_nops, or one integer (total_nops)"; + pub(crate) const parse_pack_coroutine_layout: &str = "either `no` or `captures-only`"; pub(crate) const parse_opt_panic_strategy: &str = parse_panic_strategy; pub(crate) const parse_oom_strategy: &str = "either `panic` or `abort`"; pub(crate) const parse_relro_level: &str = "one of: `full`, `partial`, or `off`"; @@ -1951,6 +1952,18 @@ pub mod parse { true } + pub(crate) fn parse_pack_coroutine_layout( + slot: &mut PackCoroutineLayout, + v: Option<&str>, + ) -> bool { + *slot = match v { + Some("no") => PackCoroutineLayout::No, + Some("captures-only") => PackCoroutineLayout::CapturesOnly, + _ => return false, + }; + true + } + pub(crate) fn parse_inlining_threshold(slot: &mut InliningThreshold, v: Option<&str>) -> bool { match v { Some("always" | "yes") => { @@ -2504,6 +2517,8 @@ options! { "panic strategy for out-of-memory handling"), osx_rpath_install_name: bool = (false, parse_bool, [TRACKED], "pass `-install_name @rpath/...` to the macOS linker (default: no)"), + pack_coroutine_layout: PackCoroutineLayout = (PackCoroutineLayout::default(), parse_pack_coroutine_layout, [TRACKED], + "set strategy to pack coroutine state layout (default: no)"), packed_bundled_libs: bool = (false, parse_bool, [TRACKED], "change rlib format to store native libraries as archives"), panic_abort_tests: bool = (false, parse_bool, [TRACKED], diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs index 136a61598d50e..66fd74ce349e4 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs @@ -24,6 +24,7 @@ use rustc_hir::{ }; use rustc_infer::infer::{BoundRegionConversionTime, DefineOpaqueTypes, InferCtxt, InferOk}; use rustc_middle::middle::privacy::Level; +use rustc_middle::mir::CoroutineLayout; use rustc_middle::traits::IsConstable; use rustc_middle::ty::error::TypeError; use rustc_middle::ty::print::{ @@ -2492,18 +2493,26 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { && let Some(coroutine_info) = self.tcx.mir_coroutine_witnesses(coroutine_did) { debug!(?coroutine_info); - 'find_source: for (variant, source_info) in - coroutine_info.variant_fields.iter().zip(&coroutine_info.variant_source_info) + 'find_source: for ((variant_idx, variant), source_info) in coroutine_info + .variant_fields + .iter_enumerated() + .zip(&coroutine_info.variant_source_info) + .rev() { debug!(?variant); for &local in variant { let decl = &coroutine_info.field_tys[local]; debug!(?decl); if ty_matches(ty::Binder::dummy(decl.ty)) && !decl.ignore_for_traits { - interior_or_upvar_span = Some(CoroutineInteriorOrUpvar::Interior( - decl.source_info.span, - Some((source_info.span, from_awaited_ty)), - )); + let span = decl.source_info.span; + if variant_idx == CoroutineLayout::UNRESUMED { + interior_or_upvar_span = Some(CoroutineInteriorOrUpvar::Upvar(span)); + } else { + interior_or_upvar_span = Some(CoroutineInteriorOrUpvar::Interior( + span, + Some((source_info.span, from_awaited_ty)), + )); + } break 'find_source; } } diff --git a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs index 2e60805cd10a5..8292f312cbcc2 100644 --- a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs +++ b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs @@ -1,6 +1,8 @@ +use rustc_abi::VariantIdx; use rustc_data_structures::fx::FxHashSet; use rustc_infer::traits::query::type_op::DropckOutlives; use rustc_middle::traits::query::{DropckConstraint, DropckOutlivesResult}; +use rustc_middle::ty::data_structures::IndexSet; use rustc_middle::ty::{self, EarlyBinder, ParamEnvAnd, Ty, TyCtxt}; use rustc_span::Span; use tracing::{debug, instrument}; @@ -354,7 +356,14 @@ pub fn dtorck_constraint_for_ty_inner<'tcx>( // FIXME(@lcnr): Why do we erase regions in the env here? Seems odd let typing_env = tcx.erase_and_anonymize_regions(typing_env); let needs_drop = tcx.mir_coroutine_witnesses(def_id).is_some_and(|witness| { - witness.field_tys.iter().any(|field| field.ty.needs_drop(tcx, typing_env)) + let upvar_locals: IndexSet<_> = + witness.variant_fields[VariantIdx::ZERO].iter().copied().collect(); + // As a reminder, upvars also appear in the UNRESUMED state-variant + witness + .field_tys + .iter_enumerated() + .filter_map(|(corsl, ty)| (!upvar_locals.contains(&corsl)).then_some(ty)) + .any(|field| field.ty.needs_drop(tcx, typing_env)) }); if needs_drop { // Pushing types directly to `constraints.outlives` is equivalent diff --git a/compiler/rustc_traits/src/coroutine_witnesses.rs b/compiler/rustc_traits/src/coroutine_witnesses.rs index 2544cd8a13cd8..5505d1646ee06 100644 --- a/compiler/rustc_traits/src/coroutine_witnesses.rs +++ b/compiler/rustc_traits/src/coroutine_witnesses.rs @@ -2,24 +2,40 @@ use rustc_infer::infer::TyCtxtInferExt; use rustc_infer::infer::canonical::query_response::make_query_region_constraints; use rustc_infer::infer::resolve::OpportunisticRegionResolver; use rustc_infer::traits::{Obligation, ObligationCause}; +use rustc_middle::mir::CoroutineLayout; +use rustc_middle::ty::data_structures::IndexSet; use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeVisitableExt, fold_regions}; use rustc_span::def_id::DefId; use rustc_trait_selection::traits::{ObligationCtxt, with_replaced_escaping_bound_vars}; +use tracing::instrument; /// Return the set of types that should be taken into account when checking /// trait bounds on a coroutine's internal state. This properly replaces /// `ReErased` with new existential bound lifetimes. +#[instrument(level = "debug", skip(tcx), ret)] pub(crate) fn coroutine_hidden_types<'tcx>( tcx: TyCtxt<'tcx>, def_id: DefId, ) -> ty::EarlyBinder<'tcx, ty::Binder<'tcx, ty::CoroutineWitnessTypes>>> { - let coroutine_layout = tcx.mir_coroutine_witnesses(def_id); + let Some(coroutine_layout) = tcx.mir_coroutine_witnesses(def_id) else { + return ty::EarlyBinder::bind(ty::Binder::dummy(ty::CoroutineWitnessTypes { + types: ty::List::empty(), + assumptions: ty::List::empty(), + })); + }; + let mut internal_fields: IndexSet<_> = + coroutine_layout.variant_fields.iter().skip(1).flat_map(|f| f.iter().copied()).collect(); + for upvar in &coroutine_layout.variant_fields[CoroutineLayout::UNRESUMED] { + internal_fields.swap_remove(upvar); + } let mut vars = vec![]; let bound_tys = tcx.mk_type_list_from_iter( - coroutine_layout - .as_ref() - .map_or_else(|| [].iter(), |l| l.field_tys.iter()) - .filter(|decl| !decl.ignore_for_traits) + internal_fields + .into_iter() + .filter_map(|saved_local| { + let saved_ty = &coroutine_layout.field_tys[saved_local]; + (!saved_ty.ignore_for_traits).then_some(saved_ty) + }) .map(|decl| { let ty = fold_regions(tcx, decl.ty, |re, debruijn| { assert_eq!(re, tcx.lifetimes.re_erased); diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index b3c0953b8e147..d0624a0ab23b8 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -11,6 +11,7 @@ use rustc_hir::attrs::AttributeKind; use rustc_hir::find_attr; use rustc_index::IndexVec; use rustc_middle::bug; +use rustc_middle::mir::CoroutineSavedLocal; use rustc_middle::query::Providers; use rustc_middle::traits::ObligationCause; use rustc_middle::ty::layout::{ @@ -20,6 +21,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{ self, AdtDef, CoroutineArgsExt, EarlyBinder, PseudoCanonicalInput, Ty, TyCtxt, TypeVisitableExt, }; +use rustc_session::config::PackCoroutineLayout; use rustc_session::{DataTypeKind, FieldInfo, FieldKind, SizeKind, VariantInfo}; use rustc_span::{Symbol, sym}; use tracing::{debug, instrument}; @@ -175,6 +177,7 @@ fn extract_const_value<'tcx>( } } +#[instrument(level = "debug", skip(cx), ret)] fn layout_of_uncached<'tcx>( cx: &LayoutCx<'tcx>, ty: Ty<'tcx>, @@ -515,8 +518,9 @@ fn layout_of_uncached<'tcx>( use rustc_middle::ty::layout::PrimitiveExt as _; let info = tcx.coroutine_layout(def_id, args)?; + debug!("info = {info:#?}"); - let local_layouts = info + let local_layouts: IndexVec<_, _> = info .field_tys .iter() .map(|local| { @@ -524,22 +528,31 @@ fn layout_of_uncached<'tcx>( let uninit_ty = Ty::new_maybe_uninit(tcx, field_ty.instantiate(tcx, args)); cx.spanned_layout_of(uninit_ty, local.source_info.span) }) - .try_collect::>()?; + .try_collect()?; - let prefix_layouts = args + let relocated_upvars = IndexVec::from_fn_n( + |local: CoroutineSavedLocal| info.relocated_upvars.get(&local).copied(), + info.field_tys.len(), + ); + let pack = match info.pack { + PackCoroutineLayout::No => rustc_abi::PackCoroutineLayout::Classic, + PackCoroutineLayout::CapturesOnly => rustc_abi::PackCoroutineLayout::CapturesOnly, + }; + let upvar_layouts = args .as_coroutine() - .prefix_tys() + .upvar_tys() .iter() .map(|ty| cx.layout_of(ty)) - .try_collect::>()?; - + .collect::>()?; let layout = cx .calc .coroutine( &local_layouts, - prefix_layouts, + &relocated_upvars, + upvar_layouts, &info.variant_fields, &info.storage_conflicts, + pack, |tag| TyAndLayout { ty: tag.primitive().to_ty(tcx), layout: tcx.mk_layout(LayoutData::scalar(cx, tag)), @@ -938,7 +951,11 @@ fn variant_info_for_coroutine<'tcx>( .then(|| Symbol::intern(&field_layout.ty.to_string())), } }) - .chain(upvar_fields.iter().copied()) + .chain( + if variant_idx == FIRST_VARIANT { &upvar_fields[..] } else { &[] } + .iter() + .copied(), + ) .collect(); // If the variant has no state-specific fields, then it's the size of the upvars. diff --git a/src/tools/clippy/tests/ui/explicit_write_in_test.stderr b/src/tools/clippy/tests/ui/explicit_write_in_test.stderr deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/tools/clippy/tests/ui/future_not_send.stderr b/src/tools/clippy/tests/ui/future_not_send.stderr index e366dc2d21958..8521eb13f62d7 100644 --- a/src/tools/clippy/tests/ui/future_not_send.stderr +++ b/src/tools/clippy/tests/ui/future_not_send.stderr @@ -93,10 +93,10 @@ LL | pub async fn public_future(&self) { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ future returned by `public_future` is not `Send` | note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> tests/ui/future_not_send.rs:49:32 + --> tests/ui/future_not_send.rs:49:33 | LL | pub async fn public_future(&self) { - | ^^^^^ has type `&Dummy` which is not `Send`, because `Dummy` is not `Sync` + | ^^^^ has type `&Dummy` which is not `Send`, because `Dummy` is not `Sync` = note: `std::rc::Rc<[u8]>` doesn't implement `std::marker::Sync` error: future cannot be sent between threads safely diff --git a/src/tools/clippy/tests/ui/needless_lifetimes.fixed b/src/tools/clippy/tests/ui/needless_lifetimes.fixed index 15ca409c95bd1..ef8a4a91677d0 100644 --- a/src/tools/clippy/tests/ui/needless_lifetimes.fixed +++ b/src/tools/clippy/tests/ui/needless_lifetimes.fixed @@ -3,6 +3,7 @@ #![warn(clippy::needless_lifetimes, clippy::elidable_lifetime_names)] #![allow( unused, + clippy::await_holding_lock, clippy::boxed_local, clippy::extra_unused_type_parameters, clippy::needless_pass_by_value, diff --git a/src/tools/clippy/tests/ui/needless_lifetimes.rs b/src/tools/clippy/tests/ui/needless_lifetimes.rs index af9649d729872..86da853dc238f 100644 --- a/src/tools/clippy/tests/ui/needless_lifetimes.rs +++ b/src/tools/clippy/tests/ui/needless_lifetimes.rs @@ -3,6 +3,7 @@ #![warn(clippy::needless_lifetimes, clippy::elidable_lifetime_names)] #![allow( unused, + clippy::await_holding_lock, clippy::boxed_local, clippy::extra_unused_type_parameters, clippy::needless_pass_by_value, diff --git a/src/tools/clippy/tests/ui/needless_lifetimes.stderr b/src/tools/clippy/tests/ui/needless_lifetimes.stderr index 138d0498c43e4..c7669e4d175b4 100644 --- a/src/tools/clippy/tests/ui/needless_lifetimes.stderr +++ b/src/tools/clippy/tests/ui/needless_lifetimes.stderr @@ -1,5 +1,5 @@ error: the following explicit lifetimes could be elided: 'a, 'b - --> tests/ui/needless_lifetimes.rs:19:23 + --> tests/ui/needless_lifetimes.rs:20:23 | LL | fn distinct_lifetimes<'a, 'b>(_x: &'a u8, _y: &'b u8, _z: u8) {} | ^^ ^^ ^^ ^^ @@ -13,7 +13,7 @@ LL + fn distinct_lifetimes(_x: &u8, _y: &u8, _z: u8) {} | error: the following explicit lifetimes could be elided: 'a, 'b - --> tests/ui/needless_lifetimes.rs:22:24 + --> tests/ui/needless_lifetimes.rs:23:24 | LL | fn distinct_and_static<'a, 'b>(_x: &'a u8, _y: &'b u8, _z: &'static u8) {} | ^^ ^^ ^^ ^^ @@ -25,7 +25,7 @@ LL + fn distinct_and_static(_x: &u8, _y: &u8, _z: &'static u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:33:15 + --> tests/ui/needless_lifetimes.rs:34:15 | LL | fn in_and_out<'a>(x: &'a u8, _y: u8) -> &'a u8 { | ^^ ^^ ^^ @@ -37,7 +37,7 @@ LL + fn in_and_out(x: &u8, _y: u8) -> &u8 { | error: the following explicit lifetimes could be elided: 'b - --> tests/ui/needless_lifetimes.rs:46:31 + --> tests/ui/needless_lifetimes.rs:47:31 | LL | fn multiple_in_and_out_2a<'a, 'b>(x: &'a u8, _y: &'b u8) -> &'a u8 { | ^^ ^^ @@ -49,7 +49,7 @@ LL + fn multiple_in_and_out_2a<'a>(x: &'a u8, _y: &u8) -> &'a u8 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:54:27 + --> tests/ui/needless_lifetimes.rs:55:27 | LL | fn multiple_in_and_out_2b<'a, 'b>(_x: &'a u8, y: &'b u8) -> &'b u8 { | ^^ ^^ @@ -61,7 +61,7 @@ LL + fn multiple_in_and_out_2b<'b>(_x: &u8, y: &'b u8) -> &'b u8 { | error: the following explicit lifetimes could be elided: 'b - --> tests/ui/needless_lifetimes.rs:72:26 + --> tests/ui/needless_lifetimes.rs:73:26 | LL | fn deep_reference_1a<'a, 'b>(x: &'a u8, _y: &'b u8) -> Result<&'a u8, ()> { | ^^ ^^ @@ -73,7 +73,7 @@ LL + fn deep_reference_1a<'a>(x: &'a u8, _y: &u8) -> Result<&'a u8, ()> { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:80:22 + --> tests/ui/needless_lifetimes.rs:81:22 | LL | fn deep_reference_1b<'a, 'b>(_x: &'a u8, y: &'b u8) -> Result<&'b u8, ()> { | ^^ ^^ @@ -85,7 +85,7 @@ LL + fn deep_reference_1b<'b>(_x: &u8, y: &'b u8) -> Result<&'b u8, ()> { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:90:21 + --> tests/ui/needless_lifetimes.rs:91:21 | LL | fn deep_reference_3<'a>(x: &'a u8, _y: u8) -> Result<&'a u8, ()> { | ^^ ^^ ^^ @@ -97,7 +97,7 @@ LL + fn deep_reference_3(x: &u8, _y: u8) -> Result<&u8, ()> { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:96:28 + --> tests/ui/needless_lifetimes.rs:97:28 | LL | fn where_clause_without_lt<'a, T>(x: &'a u8, _y: u8) -> Result<&'a u8, ()> | ^^ ^^ ^^ @@ -109,7 +109,7 @@ LL + fn where_clause_without_lt(x: &u8, _y: u8) -> Result<&u8, ()> | error: the following explicit lifetimes could be elided: 's - --> tests/ui/needless_lifetimes.rs:127:21 + --> tests/ui/needless_lifetimes.rs:128:21 | LL | fn self_and_out<'s>(&'s self) -> &'s u8 { | ^^ ^^ ^^ @@ -121,7 +121,7 @@ LL + fn self_and_out(&self) -> &u8 { | error: the following explicit lifetimes could be elided: 't - --> tests/ui/needless_lifetimes.rs:135:30 + --> tests/ui/needless_lifetimes.rs:136:30 | LL | fn self_and_in_out_1<'s, 't>(&'s self, _x: &'t u8) -> &'s u8 { | ^^ ^^ @@ -133,7 +133,7 @@ LL + fn self_and_in_out_1<'s>(&'s self, _x: &u8) -> &'s u8 { | error: the following explicit lifetimes could be elided: 's - --> tests/ui/needless_lifetimes.rs:143:26 + --> tests/ui/needless_lifetimes.rs:144:26 | LL | fn self_and_in_out_2<'s, 't>(&'s self, x: &'t u8) -> &'t u8 { | ^^ ^^ @@ -145,7 +145,7 @@ LL + fn self_and_in_out_2<'t>(&self, x: &'t u8) -> &'t u8 { | error: the following explicit lifetimes could be elided: 's, 't - --> tests/ui/needless_lifetimes.rs:148:29 + --> tests/ui/needless_lifetimes.rs:149:29 | LL | fn distinct_self_and_in<'s, 't>(&'s self, _x: &'t u8) {} | ^^ ^^ ^^ ^^ @@ -157,7 +157,7 @@ LL + fn distinct_self_and_in(&self, _x: &u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:172:21 + --> tests/ui/needless_lifetimes.rs:173:21 | LL | fn struct_with_lt4b<'a, 'b>(_foo: &'a Foo<'b>) -> &'b str { | ^^ ^^ @@ -169,7 +169,7 @@ LL + fn struct_with_lt4b<'b>(_foo: &Foo<'b>) -> &'b str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:188:22 + --> tests/ui/needless_lifetimes.rs:189:22 | LL | fn trait_obj_elided2<'a>(_arg: &'a dyn Drop) -> &'a str { | ^^ ^^ ^^ @@ -181,7 +181,7 @@ LL + fn trait_obj_elided2(_arg: &dyn Drop) -> &str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:199:20 + --> tests/ui/needless_lifetimes.rs:200:20 | LL | fn alias_with_lt4b<'a, 'b>(_foo: &'a FooAlias<'b>) -> &'b str { | ^^ ^^ @@ -193,7 +193,7 @@ LL + fn alias_with_lt4b<'b>(_foo: &FooAlias<'b>) -> &'b str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:204:30 + --> tests/ui/needless_lifetimes.rs:205:30 | LL | fn named_input_elided_output<'a>(_arg: &'a str) -> &str { | ^^ ^^ ^ @@ -205,7 +205,7 @@ LL + fn named_input_elided_output(_arg: &str) -> &str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:213:19 + --> tests/ui/needless_lifetimes.rs:214:19 | LL | fn trait_bound_ok<'a, T: WithLifetime<'static>>(_: &'a u8, _: T) { | ^^ ^^ @@ -217,7 +217,7 @@ LL + fn trait_bound_ok>(_: &u8, _: T) { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:249:24 + --> tests/ui/needless_lifetimes.rs:250:24 | LL | fn needless_lt<'a>(x: &'a u8) {} | ^^ ^^ @@ -229,7 +229,7 @@ LL + fn needless_lt(x: &u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:254:24 + --> tests/ui/needless_lifetimes.rs:255:24 | LL | fn needless_lt<'a>(_x: &'a u8) {} | ^^ ^^ @@ -241,7 +241,7 @@ LL + fn needless_lt(_x: &u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:285:55 + --> tests/ui/needless_lifetimes.rs:286:55 | LL | fn impl_trait_elidable_nested_anonymous_lifetimes<'a>(i: &'a i32, f: impl Fn(&i32) -> &i32) -> &'a i32 { | ^^ ^^ ^^ @@ -253,7 +253,7 @@ LL + fn impl_trait_elidable_nested_anonymous_lifetimes(i: &i32, f: impl Fn(& | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:295:26 + --> tests/ui/needless_lifetimes.rs:296:26 | LL | fn generics_elidable<'a, T: Fn(&i32) -> &i32>(i: &'a i32, f: T) -> &'a i32 { | ^^ ^^ ^^ @@ -265,7 +265,7 @@ LL + fn generics_elidable &i32>(i: &i32, f: T) -> &i32 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:308:30 + --> tests/ui/needless_lifetimes.rs:309:30 | LL | fn where_clause_elidable<'a, T>(i: &'a i32, f: T) -> &'a i32 | ^^ ^^ ^^ @@ -277,7 +277,7 @@ LL + fn where_clause_elidable(i: &i32, f: T) -> &i32 | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:324:28 + --> tests/ui/needless_lifetimes.rs:325:28 | LL | fn pointer_fn_elidable<'a>(i: &'a i32, f: fn(&i32) -> &i32) -> &'a i32 { | ^^ ^^ ^^ @@ -289,7 +289,7 @@ LL + fn pointer_fn_elidable(i: &i32, f: fn(&i32) -> &i32) -> &i32 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:338:28 + --> tests/ui/needless_lifetimes.rs:339:28 | LL | fn nested_fn_pointer_3<'a>(_: &'a i32) -> fn(fn(&i32) -> &i32) -> i32 { | ^^ ^^ @@ -301,7 +301,7 @@ LL + fn nested_fn_pointer_3(_: &i32) -> fn(fn(&i32) -> &i32) -> i32 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:342:28 + --> tests/ui/needless_lifetimes.rs:343:28 | LL | fn nested_fn_pointer_4<'a>(_: &'a i32) -> impl Fn(fn(&i32)) { | ^^ ^^ @@ -313,7 +313,7 @@ LL + fn nested_fn_pointer_4(_: &i32) -> impl Fn(fn(&i32)) { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:365:21 + --> tests/ui/needless_lifetimes.rs:366:21 | LL | fn implicit<'a>(&'a self) -> &'a () { | ^^ ^^ ^^ @@ -325,7 +325,7 @@ LL + fn implicit(&self) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:369:25 + --> tests/ui/needless_lifetimes.rs:370:25 | LL | fn implicit_mut<'a>(&'a mut self) -> &'a () { | ^^ ^^ ^^ @@ -337,7 +337,7 @@ LL + fn implicit_mut(&mut self) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:374:21 + --> tests/ui/needless_lifetimes.rs:375:21 | LL | fn explicit<'a>(self: &'a Arc) -> &'a () { | ^^ ^^ ^^ @@ -349,7 +349,7 @@ LL + fn explicit(self: &Arc) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:379:25 + --> tests/ui/needless_lifetimes.rs:380:25 | LL | fn explicit_mut<'a>(self: &'a mut Rc) -> &'a () { | ^^ ^^ ^^ @@ -361,7 +361,7 @@ LL + fn explicit_mut(self: &mut Rc) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:392:31 + --> tests/ui/needless_lifetimes.rs:393:31 | LL | fn lifetime_elsewhere<'a>(self: Box, here: &'a ()) -> &'a () { | ^^ ^^ ^^ @@ -373,7 +373,7 @@ LL + fn lifetime_elsewhere(self: Box, here: &()) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:399:21 + --> tests/ui/needless_lifetimes.rs:400:21 | LL | fn implicit<'a>(&'a self) -> &'a (); | ^^ ^^ ^^ @@ -385,7 +385,7 @@ LL + fn implicit(&self) -> &(); | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:401:30 + --> tests/ui/needless_lifetimes.rs:402:30 | LL | fn implicit_provided<'a>(&'a self) -> &'a () { | ^^ ^^ ^^ @@ -397,7 +397,7 @@ LL + fn implicit_provided(&self) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:407:21 + --> tests/ui/needless_lifetimes.rs:408:21 | LL | fn explicit<'a>(self: &'a Arc) -> &'a (); | ^^ ^^ ^^ @@ -409,7 +409,7 @@ LL + fn explicit(self: &Arc) -> &(); | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:410:30 + --> tests/ui/needless_lifetimes.rs:411:30 | LL | fn explicit_provided<'a>(self: &'a Arc) -> &'a () { | ^^ ^^ ^^ @@ -421,7 +421,7 @@ LL + fn explicit_provided(self: &Arc) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:421:31 + --> tests/ui/needless_lifetimes.rs:422:31 | LL | fn lifetime_elsewhere<'a>(self: Box, here: &'a ()) -> &'a (); | ^^ ^^ ^^ @@ -433,7 +433,7 @@ LL + fn lifetime_elsewhere(self: Box, here: &()) -> &(); | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:423:40 + --> tests/ui/needless_lifetimes.rs:424:40 | LL | fn lifetime_elsewhere_provided<'a>(self: Box, here: &'a ()) -> &'a () { | ^^ ^^ ^^ @@ -445,7 +445,7 @@ LL + fn lifetime_elsewhere_provided(self: Box, here: &()) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:433:12 + --> tests/ui/needless_lifetimes.rs:434:12 | LL | fn foo<'a>(x: &'a u8, y: &'_ u8) {} | ^^ ^^ @@ -457,7 +457,7 @@ LL + fn foo(x: &u8, y: &'_ u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:436:12 + --> tests/ui/needless_lifetimes.rs:437:12 | LL | fn bar<'a>(x: &'a u8, y: &'_ u8, z: &'_ u8) {} | ^^ ^^ @@ -469,7 +469,7 @@ LL + fn bar(x: &u8, y: &'_ u8, z: &'_ u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:444:18 + --> tests/ui/needless_lifetimes.rs:445:18 | LL | fn one_input<'a>(x: &'a u8) -> &'a u8 { | ^^ ^^ ^^ @@ -481,7 +481,7 @@ LL + fn one_input(x: &u8) -> &u8 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:450:42 + --> tests/ui/needless_lifetimes.rs:451:42 | LL | fn multiple_inputs_output_not_elided<'a, 'b>(x: &'a u8, y: &'b u8, z: &'b u8) -> &'b u8 { | ^^ ^^ @@ -493,7 +493,7 @@ LL + fn multiple_inputs_output_not_elided<'b>(x: &u8, y: &'b u8, z: &'b u8) | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:467:22 + --> tests/ui/needless_lifetimes.rs:468:22 | LL | fn one_input<'a>(x: &'a u8) -> &'a u8 { | ^^ ^^ ^^ diff --git a/src/tools/miri/tests/pass/async-drop.rs b/src/tools/miri/tests/pass/async-drop.rs index 4fa84384d9bdd..6754d4ffbef10 100644 --- a/src/tools/miri/tests/pass/async-drop.rs +++ b/src/tools/miri/tests/pass/async-drop.rs @@ -1,6 +1,8 @@ -//@revisions: stack tree -//@compile-flags: -Zmiri-strict-provenance -//@[tree]compile-flags: -Zmiri-tree-borrows +//@ revisions: stack tree stackrelocate treerelocate +//@ compile-flags: -Zmiri-strict-provenance -Zpack-coroutine-layout=no +//@ [tree] compile-flags: -Zmiri-tree-borrows +//@ [stackrelocate] compile-flags: -Zpack-coroutine-layout=captures-only +//@ [treerelocate] compile-flags: -Zmiri-tree-borrows -Zpack-coroutine-layout=captures-only // WARNING: If you would ever want to modify this test, // please consider modifying rustc's async drop test at diff --git a/src/tools/miri/tests/pass/async-drop.stackrelocate.stdout b/src/tools/miri/tests/pass/async-drop.stackrelocate.stdout new file mode 100644 index 0000000000000..fc53df2f1b485 --- /dev/null +++ b/src/tools/miri/tests/pass/async-drop.stackrelocate.stdout @@ -0,0 +1,23 @@ +AsyncInt::async_drop: 0 +AsyncInt::async_drop: 1 +AsyncInt::async_drop: 2 +AsyncInt::async_drop: 3 +AsyncInt::async_drop: 4 +AsyncStruct::async_drop: 6 +AsyncInt::async_drop: 7 +AsyncInt::async_drop: 8 +AsyncReference::async_drop: 10 +AsyncInt::async_drop: 11 +AsyncEnum(A)::async_drop: 12 +SyncInt::drop: 12 +AsyncEnum(B)::async_drop: 13 +AsyncInt::async_drop: 13 +SyncInt::drop: 14 +SyncThenAsync::drop: 15 +AsyncInt::async_drop: 16 +SyncInt::drop: 17 +AsyncInt::async_drop: 18 +AsyncInt::async_drop: 19 +AsyncInt::async_drop: 20 +AsyncUnion::async_drop: 21, 21 +AsyncInt::async_drop: 10 diff --git a/src/tools/miri/tests/pass/async-drop.treerelocate.stdout b/src/tools/miri/tests/pass/async-drop.treerelocate.stdout new file mode 100644 index 0000000000000..fc53df2f1b485 --- /dev/null +++ b/src/tools/miri/tests/pass/async-drop.treerelocate.stdout @@ -0,0 +1,23 @@ +AsyncInt::async_drop: 0 +AsyncInt::async_drop: 1 +AsyncInt::async_drop: 2 +AsyncInt::async_drop: 3 +AsyncInt::async_drop: 4 +AsyncStruct::async_drop: 6 +AsyncInt::async_drop: 7 +AsyncInt::async_drop: 8 +AsyncReference::async_drop: 10 +AsyncInt::async_drop: 11 +AsyncEnum(A)::async_drop: 12 +SyncInt::drop: 12 +AsyncEnum(B)::async_drop: 13 +AsyncInt::async_drop: 13 +SyncInt::drop: 14 +SyncThenAsync::drop: 15 +AsyncInt::async_drop: 16 +SyncInt::drop: 17 +AsyncInt::async_drop: 18 +AsyncInt::async_drop: 19 +AsyncInt::async_drop: 20 +AsyncUnion::async_drop: 21, 21 +AsyncInt::async_drop: 10 diff --git a/tests/codegen-llvm/async-fn-debug.rs b/tests/codegen-llvm/async-fn-debug.rs index ed704c7cc8b92..a1fc4f14db0b1 100644 --- a/tests/codegen-llvm/async-fn-debug.rs +++ b/tests/codegen-llvm/async-fn-debug.rs @@ -38,17 +38,17 @@ async fn async_fn_test() { // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) // CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "3", scope: [[VARIANT]], -// CHECK-SAME: file: [[FILE]], line: 13, +// CHECK-SAME: file: [[FILE]], line: 15, // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) -// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "4", scope: [[VARIANT]], -// CHECK-SAME: file: [[FILE]], line: 15, +// CHECK: [[S0:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "Suspend0", scope: [[GEN]], // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) -// CHECK: [[S1:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "Suspend1", scope: [[GEN]], +// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "s", scope: [[S0]] // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) -// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "s", scope: [[S1]] +// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "4", scope: [[VARIANT]], +// CHECK-SAME: file: [[FILE]], line: 13, // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) // CHECK: [[DISC]] = !DIDerivedType(tag: DW_TAG_member, name: "__state", scope: [[GEN]], diff --git a/tests/debuginfo/coroutine-objects.rs b/tests/debuginfo/coroutine-objects.rs index f1165de4bfaaf..f26702b0f9f1b 100644 --- a/tests/debuginfo/coroutine-objects.rs +++ b/tests/debuginfo/coroutine-objects.rs @@ -4,7 +4,7 @@ // with memory layout of discriminant for this particular enum // ensure that LLDB won't crash at least (like #57822). -//@ compile-flags:-g +//@ compile-flags: -g -Z pack-coroutine-layout=no //@ disable-gdb-pretty-printers //@ ignore-backends: gcc @@ -12,22 +12,22 @@ // gdb-command:run // gdb-command:print b -// gdb-check:$1 = coroutine_objects::main::{coroutine_env#0}::Unresumed{_ref__a: 0x[...]} +// gdb-check:$1 = coroutine_objects::main::{coroutine_env#0}::Unresumed{a: 0x[...]} // gdb-command:continue // gdb-command:print b -// gdb-check:$2 = coroutine_objects::main::{coroutine_env#0}::Suspend0{c: 6, d: 7, _ref__a: 0x[...]} +// gdb-check:$2 = coroutine_objects::main::{coroutine_env#0}::Suspend0{c: 6, d: 7} // gdb-command:continue // gdb-command:print b -// gdb-check:$3 = coroutine_objects::main::{coroutine_env#0}::Suspend1{c: 7, d: 8, _ref__a: 0x[...]} +// gdb-check:$3 = coroutine_objects::main::{coroutine_env#0}::Suspend1{c: 7, d: 8} // gdb-command:continue // gdb-command:print b -// gdb-check:$4 = coroutine_objects::main::{coroutine_env#0}::Returned{_ref__a: 0x[...]} +// gdb-check:$4 = coroutine_objects::main::{coroutine_env#0}::Returned // === LLDB TESTS ================================================================================== // lldb-command:run // lldb-command:v b -// lldb-check:(coroutine_objects::main::{coroutine_env#0}) b = { value = { _ref__a = 0x[...] } $discr$ = [...] } +// lldb-check:(coroutine_objects::main::{coroutine_env#0}) b = { value = { a = 0x[...] } $discr$ = [...] } // === CDB TESTS =================================================================================== diff --git a/tests/debuginfo/coroutine-objects_relocated.rs b/tests/debuginfo/coroutine-objects_relocated.rs new file mode 100644 index 0000000000000..83113f89607ed --- /dev/null +++ b/tests/debuginfo/coroutine-objects_relocated.rs @@ -0,0 +1,89 @@ +//@ min-lldb-version: 1800 + +// LLDB (18.1+) now supports DW_TAG_variant_part, but there is some bug in either compiler or LLDB +// with memory layout of discriminant for this particular enum +// ensure that LLDB won't crash at least (like #57822). + +//@ compile-flags: -g -Z pack-coroutine-layout=captures-only +//@ disable-gdb-pretty-printers + +// === GDB TESTS =================================================================================== + +// gdb-command:run +// gdb-command:print b +// gdb-check:$1 = coroutine_objects_relocated::main::{coroutine_env#0}::Unresumed{a: 0x[...]} +// gdb-command:continue +// gdb-command:print b +// gdb-check:$2 = coroutine_objects_relocated::main::{coroutine_env#0}::Suspend0{c: 6, d: 7, a: 0x[...]} +// gdb-command:continue +// gdb-command:print b +// gdb-check:$3 = coroutine_objects_relocated::main::{coroutine_env#0}::Suspend1{c: 7, d: 8, a: 0x[...]} +// gdb-command:continue +// gdb-command:print b +// gdb-check:$4 = coroutine_objects_relocated::main::{coroutine_env#0}::Returned + +// === LLDB TESTS ================================================================================== + +// lldb-command:run +// lldb-command:v b +// lldb-check:(coroutine_objects_relocated::main::{coroutine_env#0}) b = { value = { a = 0x[...] } $discr$ = [...] } + +// === CDB TESTS =================================================================================== + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Unresumed [Type: enum2$] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 5 [Type: int *] + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Suspend0 [Type: enum2$] +// cdb-check: [+0x[...]] c : 6 [Type: int] +// cdb-check: [+0x[...]] d : 7 [Type: int] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 5 [Type: int *] + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Suspend1 [Type: enum2$] +// cdb-check: [+0x[...]] c : 7 [Type: int] +// cdb-check: [+0x[...]] d : 8 [Type: int] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 6 [Type: int *] + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Returned [Type: enum2$] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 6 [Type: int *] + +#![feature(coroutines, coroutine_trait, stmt_expr_attributes)] + +use std::ops::Coroutine; +use std::pin::Pin; + +fn main() { + let mut a = 5; + let mut b = #[coroutine] + || { + let mut c = 6; + let mut d = 7; + + yield; + a += 1; + c += 1; + d += 1; + + yield; + println!("{} {} {}", a, c, d); + }; + _zzz(); // #break + Pin::new(&mut b).resume(()); + _zzz(); // #break + Pin::new(&mut b).resume(()); + _zzz(); // #break + Pin::new(&mut b).resume(()); + _zzz(); // #break +} + +#[inline(never)] +fn _zzz() { + () +} diff --git a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir index 435c1532895ca..b2bcc68428b07 100644 --- a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir +++ b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir @@ -2,7 +2,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> { debug _task_context => _2; - debug x => ((*_20).0: T); + debug x => (((*_20) as variant#0).0: T); let mut _0: std::task::Poll<()>; let _3: T; let mut _4: impl std::future::Future; @@ -83,7 +83,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) } bb11: { - drop(((*_20).0: T)) -> [return: bb10, unwind unreachable]; + drop((((*_20) as variant#0).0: T)) -> [return: bb10, unwind unreachable]; } bb12: { diff --git a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir index 1dc1d08136290..f0b94254fe26e 100644 --- a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir +++ b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir @@ -2,7 +2,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> { debug _task_context => _2; - debug x => ((*_20).0: T); + debug x => (((*_20) as variant#0).0: T); let mut _0: std::task::Poll<()>; let _3: T; let mut _4: impl std::future::Future; @@ -97,7 +97,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) } bb14: { - drop(((*_20).0: T)) -> [return: bb13, unwind: bb4]; + drop((((*_20) as variant#0).0: T)) -> [return: bb13, unwind: bb4]; } bb15 (cleanup): { diff --git a/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir index 6cad5b105d3e3..55fd5920e658f 100644 --- a/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir @@ -7,6 +7,7 @@ Panicked (2): [], }, storage_conflicts: BitMatrix(0x0) {}, + relocated_upvars: {}, } */ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> { diff --git a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir index 96ee37185db16..6ecd583035a56 100644 --- a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir @@ -40,13 +40,14 @@ Unresumed(0): [], Returned (1): [], Panicked (2): [], - Suspend0 (3): [_s0], - Suspend1 (4): [_s1], + Suspend0 (3): [_s1], + Suspend1 (4): [_s0], }, storage_conflicts: BitMatrix(2x2) { (_s0, _s0), (_s1, _s1), }, + relocated_upvars: {}, } */ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> Poll<()> { @@ -88,14 +89,14 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> let mut _38: u32; let mut _39: &mut {async fn body of b()}; scope 1 { - debug __awaitee => (((*_39) as variant#3).0: {async fn body of a()}); + debug __awaitee => (((*_39) as variant#4).0: {async fn body of a()}); let _17: (); scope 2 { debug result => _17; } } scope 3 { - debug __awaitee => (((*_39) as variant#4).0: {async fn body of a()}); + debug __awaitee => (((*_39) as variant#3).0: {async fn body of a()}); let _33: (); scope 4 { debug result => _33; @@ -123,7 +124,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_5); PlaceMention(_4); nop; - (((*_39) as variant#3).0: {async fn body of a()}) = move _4; + (((*_39) as variant#4).0: {async fn body of a()}) = move _4; goto -> bb4; } @@ -133,7 +134,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageLive(_10); StorageLive(_11); StorageLive(_12); - _12 = &mut (((*_39) as variant#3).0: {async fn body of a()}); + _12 = &mut (((*_39) as variant#4).0: {async fn body of a()}); _11 = &mut (*_12); _10 = Pin::<&mut {async fn body of a()}>::new_unchecked(move _11) -> [return: bb5, unwind unreachable]; } @@ -193,7 +194,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_12); StorageDead(_9); StorageDead(_8); - drop((((*_39) as variant#3).0: {async fn body of a()})) -> [return: bb12, unwind unreachable]; + drop((((*_39) as variant#4).0: {async fn body of a()})) -> [return: bb12, unwind unreachable]; } bb11: { @@ -225,7 +226,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_22); PlaceMention(_21); nop; - (((*_39) as variant#4).0: {async fn body of a()}) = move _21; + (((*_39) as variant#3).0: {async fn body of a()}) = move _21; goto -> bb16; } @@ -235,7 +236,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageLive(_26); StorageLive(_27); StorageLive(_28); - _28 = &mut (((*_39) as variant#4).0: {async fn body of a()}); + _28 = &mut (((*_39) as variant#3).0: {async fn body of a()}); _27 = &mut (*_28); _26 = Pin::<&mut {async fn body of a()}>::new_unchecked(move _27) -> [return: bb17, unwind unreachable]; } @@ -290,7 +291,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_28); StorageDead(_25); StorageDead(_24); - drop((((*_39) as variant#4).0: {async fn body of a()})) -> [return: bb23, unwind unreachable]; + drop((((*_39) as variant#3).0: {async fn body of a()})) -> [return: bb23, unwind unreachable]; } bb22: { diff --git a/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir b/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir index b61215dc28cb4..40f0facd5c48f 100644 --- a/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir +++ b/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir @@ -20,6 +20,7 @@ storage_conflicts: BitMatrix(1x1) { (_s0, _s0), }, + relocated_upvars: {}, } */ fn main::{closure#0}(_1: Pin<&mut {coroutine@$DIR/coroutine.rs:18:5: 18:18}>, _2: String) -> CoroutineState<(&str, String, &Location<'_>), ()> { diff --git a/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir b/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir index aac028a9e6c0e..07a5d7b4c89d2 100644 --- a/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir +++ b/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir @@ -20,6 +20,7 @@ storage_conflicts: BitMatrix(1x1) { (_s0, _s0), }, + relocated_upvars: {}, } */ fn main::{closure#1}(_1: Pin<&mut {coroutine@$DIR/coroutine.rs:25:5: 25:18}>, _2: String) -> CoroutineState<(&str, String, &Location<'_>), ()> { diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-abort.diff b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-abort.diff new file mode 100644 index 0000000000000..aec81b219d4a6 --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-abort.diff @@ -0,0 +1,91 @@ +- // MIR for `main::{closure#0}` before RelocateUpvars ++ // MIR for `main::{closure#0}` after RelocateUpvars + + fn main::{closure#0}(_1: {coroutine@$DIR/coroutine_relocate_upvars.rs:11:5: 11:7}, _2: ()) -> () + yields () + { +- debug x => (*(_1.0: &mut std::string::String)); ++ debug x => (*_9); ++ debug x => _9; + let mut _0: (); + let mut _3: std::string::String; + let _4: (); + let mut _5: (); + let mut _6: &mut std::string::String; + let mut _7: &mut std::string::String; + let mut _8: &mut std::string::String; ++ let mut _9: &mut std::string::String; ++ let mut _10: &mut std::string::String; + + bb0: { ++ StorageLive(_9); ++ StorageLive(_10); ++ _10 = move (_1.0: &mut std::string::String); ++ _9 = move _10; ++ StorageDead(_10); + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind unreachable]; + } + + bb1: { +- _6 = copy (_1.0: &mut std::string::String); ++ _6 = copy _9; + drop((*_6)) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- _7 = copy (_1.0: &mut std::string::String); ++ _7 = copy _9; + (*_7) = move _3; + goto -> bb3; + } + + bb3: { + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = (); + _4 = yield(move _5) -> [resume: bb4, drop: bb6]; + } + + bb4: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); +- drop(_1) -> [return: bb5, unwind unreachable]; ++ goto -> bb10; + } + + bb5: { ++ StorageDead(_9); + return; + } + + bb6: { + StorageDead(_5); + StorageDead(_4); + goto -> bb7; + } + + bb7: { +- drop(_1) -> [return: bb8, unwind unreachable]; ++ goto -> bb11; + } + + bb8: { + coroutine_drop; ++ } ++ ++ bb9 (cleanup): { ++ unreachable; ++ } ++ ++ bb10: { ++ goto -> bb5; ++ } ++ ++ bb11: { ++ goto -> bb8; + } + } + diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-unwind.diff b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-unwind.diff new file mode 100644 index 0000000000000..3ed867e37ca70 --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-unwind.diff @@ -0,0 +1,104 @@ +- // MIR for `main::{closure#0}` before RelocateUpvars ++ // MIR for `main::{closure#0}` after RelocateUpvars + + fn main::{closure#0}(_1: {coroutine@$DIR/coroutine_relocate_upvars.rs:11:5: 11:7}, _2: ()) -> () + yields () + { +- debug x => (*(_1.0: &mut std::string::String)); ++ debug x => (*_9); ++ debug x => _9; + let mut _0: (); + let mut _3: std::string::String; + let _4: (); + let mut _5: (); + let mut _6: &mut std::string::String; + let mut _7: &mut std::string::String; + let mut _8: &mut std::string::String; ++ let mut _9: &mut std::string::String; ++ let mut _10: &mut std::string::String; + + bb0: { ++ StorageLive(_9); ++ StorageLive(_10); ++ _10 = move (_1.0: &mut std::string::String); ++ _9 = move _10; ++ StorageDead(_10); + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind: bb10]; + } + + bb1: { +- _6 = copy (_1.0: &mut std::string::String); ++ _6 = copy _9; + drop((*_6)) -> [return: bb2, unwind: bb3]; + } + + bb2: { +- _7 = copy (_1.0: &mut std::string::String); ++ _7 = copy _9; + (*_7) = move _3; + goto -> bb4; + } + + bb3 (cleanup): { +- _8 = copy (_1.0: &mut std::string::String); ++ _8 = copy _9; + (*_8) = move _3; + goto -> bb10; + } + + bb4: { + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = (); + _4 = yield(move _5) -> [resume: bb5, drop: bb7]; + } + + bb5: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); +- drop(_1) -> [return: bb6, unwind: bb11]; ++ goto -> bb12; + } + + bb6: { ++ StorageDead(_9); + return; + } + + bb7: { + StorageDead(_5); + StorageDead(_4); + goto -> bb8; + } + + bb8: { +- drop(_1) -> [return: bb9, unwind: bb11]; ++ goto -> bb13; + } + + bb9: { + coroutine_drop; + } + + bb10 (cleanup): { + StorageDead(_3); +- drop(_1) -> [return: bb11, unwind terminate(cleanup)]; ++ goto -> bb11; + } + + bb11 (cleanup): { + resume; ++ } ++ ++ bb12: { ++ goto -> bb6; ++ } ++ ++ bb13: { ++ goto -> bb9; + } + } + diff --git a/tests/mir-opt/coroutine_relocate_upvars.rs b/tests/mir-opt/coroutine_relocate_upvars.rs new file mode 100644 index 0000000000000..e65123d93e99c --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.rs @@ -0,0 +1,21 @@ +// skip-filecheck +//@ compile-flags: -Zpack-coroutine-layout=captures-only +#![feature(coroutines, coroutine_trait, stmt_expr_attributes)] + +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY + +// EMIT_MIR coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.diff +fn main() { + let mut x = String::new(); + let capture_by_ref = #[coroutine] + || { + x = String::new(); + yield; + }; + let mut y = String::new(); + let capture_by_val = #[coroutine] + || { + y = String::new(); + yield; + }; +} diff --git a/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir index 222c7144ef07d..242afe469526b 100644 --- a/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir @@ -19,6 +19,7 @@ storage_conflicts: BitMatrix(1x1) { (_s0, _s0), }, + relocated_upvars: {}, } */ fn main::{closure#0}(_1: Pin<&mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13}>, _2: u8) -> CoroutineState<(), ()> { diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff index 4c1b25c786efc..5b5d99fcbedca 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff @@ -154,7 +154,7 @@ } + bb3: { -+ (((*_32) as variant#3).0: ActionPermit<'_, T>) = move ((*_32).0: ActionPermit<'_, T>); ++ (((*_32) as variant#3).0: ActionPermit<'_, T>) = move (((*_32) as variant#0).0: ActionPermit<'_, T>); + StorageLive(_12); + StorageLive(_13); + StorageLive(_14); diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff index 59417ce646513..0dc7973dddaa2 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff @@ -165,7 +165,7 @@ - StorageDead(_2); - return; + bb5: { -+ (((*_32) as variant#3).0: ActionPermit<'_, T>) = move ((*_32).0: ActionPermit<'_, T>); ++ (((*_32) as variant#3).0: ActionPermit<'_, T>) = move (((*_32) as variant#0).0: ActionPermit<'_, T>); + StorageLive(_12); + StorageLive(_13); + StorageLive(_14); diff --git a/tests/ui/async-await/async-drop/async-drop-initial.run.stdout b/tests/ui/async-await/async-drop/async-drop-initial.classic.run.stdout similarity index 94% rename from tests/ui/async-await/async-drop/async-drop-initial.run.stdout rename to tests/ui/async-await/async-drop/async-drop-initial.classic.run.stdout index 9cae4331caf92..792880d835935 100644 --- a/tests/ui/async-await/async-drop/async-drop-initial.run.stdout +++ b/tests/ui/async-await/async-drop/async-drop-initial.classic.run.stdout @@ -20,3 +20,4 @@ AsyncInt::Dropper::poll: 18 AsyncInt::Dropper::poll: 19 AsyncInt::Dropper::poll: 20 AsyncUnion::Dropper::poll: 21, 21 +AsyncUnion::Dropper::poll: 21, 21 diff --git a/tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout b/tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout new file mode 100644 index 0000000000000..792880d835935 --- /dev/null +++ b/tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout @@ -0,0 +1,23 @@ +AsyncInt::Dropper::poll: 0 +AsyncInt::Dropper::poll: 1 +AsyncInt::Dropper::poll: 2 +AsyncInt::Dropper::poll: 3 +AsyncInt::Dropper::poll: 4 +AsyncStruct::Dropper::poll: 6 +AsyncInt::Dropper::poll: 7 +AsyncInt::Dropper::poll: 8 +AsyncReference::Dropper::poll: 10 +AsyncInt::Dropper::poll: 11 +AsyncEnum(A)::Dropper::poll: 12 +SyncInt::drop: 12 +AsyncEnum(B)::Dropper::poll: 13 +AsyncInt::Dropper::poll: 13 +SyncInt::drop: 14 +SyncThenAsync::drop: 15 +AsyncInt::Dropper::poll: 16 +SyncInt::drop: 17 +AsyncInt::Dropper::poll: 18 +AsyncInt::Dropper::poll: 19 +AsyncInt::Dropper::poll: 20 +AsyncUnion::Dropper::poll: 21, 21 +AsyncUnion::Dropper::poll: 21, 21 diff --git a/tests/ui/async-await/async-drop/async-drop-initial.rs b/tests/ui/async-await/async-drop/async-drop-initial.rs index cd33c143fba01..ba76d0e951f25 100644 --- a/tests/ui/async-await/async-drop/async-drop-initial.rs +++ b/tests/ui/async-await/async-drop/async-drop-initial.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ run-pass //@ check-run-results @@ -11,26 +14,32 @@ //@ edition: 2021 // FIXME(zetanumbers): consider AsyncDestruct::async_drop cleanup tests -use core::future::{async_drop_in_place, AsyncDrop, Future}; +use core::future::{AsyncDrop, Future, async_drop_in_place}; use core::hint::black_box; use core::mem::{self, ManuallyDrop}; -use core::pin::{pin, Pin}; +use core::pin::{Pin, pin}; +use core::sync::atomic::{AtomicBool, Ordering}; use core::task::{Context, Poll, Waker}; -async fn test_async_drop(x: T, _size: usize) { +static PASS: AtomicBool = AtomicBool::new(false); + +async fn test_async_drop(x: T, #[allow(unused)] expect: usize) { let mut x = mem::MaybeUninit::new(x); let dtor = pin!(unsafe { async_drop_in_place(x.as_mut_ptr()) }); // FIXME(zetanumbers): This check fully depends on the layout of // the coroutine state, since async destructor combinators are just // async functions. + #[allow(unused)] + let got = mem::size_of_val(&*dtor); #[cfg(target_pointer_width = "64")] - assert_eq!( - mem::size_of_val(&*dtor), - _size, - "sizes did not match for async destructor of type {}", - core::any::type_name::(), - ); + if expect != got { + println!( + "sizes did not match for async destructor of type {}, expect {expect}, got {got}", + core::any::type_name::() + ); + PASS.store(false, Ordering::Relaxed); + } test_idempotency(dtor).await; } @@ -47,6 +56,7 @@ where } fn main() { + PASS.store(true, Ordering::Relaxed); let waker = Waker::noop(); let mut cx = Context::from_waker(&waker); @@ -54,16 +64,17 @@ fn main() { let fut = pin!(async { test_async_drop(Int(0), 16).await; test_async_drop(AsyncInt(0), 32).await; - test_async_drop([AsyncInt(1), AsyncInt(2)], 104).await; - test_async_drop((AsyncInt(3), AsyncInt(4)), 120).await; + test_async_drop([AsyncInt(1), AsyncInt(2)], [96, 104][cfg!(classic) as usize]).await; + test_async_drop((AsyncInt(3), AsyncInt(4)), [112, 120][cfg!(classic) as usize]).await; test_async_drop(5, 16).await; let j = 42; test_async_drop(&i, 16).await; test_async_drop(&j, 16).await; test_async_drop( AsyncStruct { b: AsyncInt(8), a: AsyncInt(7), i: 6 }, - 136, - ).await; + if cfg!(panic = "unwind") { [128, 136][cfg!(classic) as usize] } else { 136 }, + ) + .await; test_async_drop(ManuallyDrop::new(AsyncInt(9)), 16).await; let foo = AsyncInt(10); @@ -81,13 +92,13 @@ fn main() { ) .await; - test_async_drop(AsyncEnum::A(AsyncInt(12)), 104).await; - test_async_drop(AsyncEnum::B(SyncInt(13)), 104).await; + test_async_drop(AsyncEnum::A(AsyncInt(12)), [96, 104][cfg!(classic) as usize]).await; + test_async_drop(AsyncEnum::B(SyncInt(13)), [96, 104][cfg!(classic) as usize]).await; test_async_drop(SyncInt(14), 16).await; test_async_drop( SyncThenAsync { i: 15, a: AsyncInt(16), b: SyncInt(17), c: AsyncInt(18) }, - 120, + [112, 120][cfg!(classic) as usize], ) .await; @@ -109,9 +120,11 @@ fn main() { .await; test_async_drop(AsyncUnion { signed: 21 }, 32).await; + test_async_drop(AsyncUnion { signed: 21 }, 32).await; }); let res = fut.poll(&mut cx); assert_eq!(res, Poll::Ready(())); + assert!(PASS.load(Ordering::Relaxed)); } struct AsyncInt(i32); @@ -228,19 +241,13 @@ union AsyncUnion { impl Drop for AsyncUnion { fn drop(&mut self) { - println!( - "AsyncUnion::drop: {}, {}", - unsafe { self.signed }, - unsafe { self.unsigned }, - ); + println!("AsyncUnion::drop: {}, {}", unsafe { self.signed }, unsafe { self.unsigned },); } } impl AsyncDrop for AsyncUnion { async fn drop(self: Pin<&mut Self>) { - println!( - "AsyncUnion::Dropper::poll: {}, {}", - unsafe { self.signed }, - unsafe { self.unsigned }, - ); + println!("AsyncUnion::Dropper::poll: {}, {}", unsafe { self.signed }, unsafe { + self.unsigned + },); } } diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout b/tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout similarity index 88% rename from tests/ui/async-await/future-sizes/async-awaiting-fut.stdout rename to tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout index b30c15bcbe6ed..c8363ec90e03e 100644 --- a/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout @@ -14,26 +14,24 @@ print-type-size field `.value`: 3077 bytes print-type-size type: `{async fn body of calls_fut<{async fn body of big_fut()}>()}`: 3077 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes -print-type-size variant `Suspend0`: 2052 bytes -print-type-size upvar `.fut`: 1025 bytes print-type-size local `.fut`: 1025 bytes +print-type-size upvar `.fut`: 1025 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 2052 bytes +print-type-size padding: 1025 bytes +print-type-size local `.fut`: 1025 bytes, alignment: 1 bytes print-type-size local `..coroutine_field4`: 1 bytes, type: bool print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} print-type-size variant `Suspend1`: 3076 bytes -print-type-size upvar `.fut`: 1025 bytes -print-type-size padding: 1025 bytes +print-type-size padding: 2050 bytes print-type-size local `..coroutine_field4`: 1 bytes, alignment: 1 bytes, type: bool print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of big_fut()} print-type-size variant `Suspend2`: 2052 bytes -print-type-size upvar `.fut`: 1025 bytes -print-type-size local `.fut`: 1025 bytes +print-type-size padding: 1025 bytes +print-type-size local `.fut`: 1025 bytes, alignment: 1 bytes print-type-size local `..coroutine_field4`: 1 bytes, type: bool print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} print-type-size variant `Returned`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes print-type-size variant `Panicked`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes print-type-size field `.value`: 1025 bytes print-type-size type: `std::mem::MaybeUninit<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes @@ -43,11 +41,16 @@ print-type-size field `.value`: 1025 bytes print-type-size type: `{async fn body of big_fut()}`: 1025 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes +print-type-size local `.arg`: 1024 bytes +print-type-size upvar `.arg`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes print-type-size field `.waker`: 8 bytes print-type-size field `.local_waker`: 8 bytes diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout b/tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout new file mode 100644 index 0000000000000..2c658c5f925a0 --- /dev/null +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout @@ -0,0 +1,111 @@ +print-type-size type: `{async fn body of test()}`: 2053 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Suspend0`: 2052 bytes +print-type-size local `.__awaitee`: 2052 bytes, type: {async fn body of calls_fut<{async fn body of big_fut()}>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 2052 bytes, alignment: 1 bytes +print-type-size field `.value`: 2052 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 2052 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 2052 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 2052 bytes +print-type-size type: `{async fn body of calls_fut<{async fn body of big_fut()}>()}`: 2052 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 2051 bytes +print-type-size upvar `.fut`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size padding: 1026 bytes +print-type-size local `.fut`: 1025 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 1027 bytes +print-type-size local `.fut`: 1025 bytes +print-type-size local `..coroutine_field4`: 1 bytes, type: bool +print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} +print-type-size variant `Suspend1`: 2051 bytes +print-type-size padding: 1025 bytes +print-type-size local `..coroutine_field4`: 1 bytes, alignment: 1 bytes, type: bool +print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of big_fut()} +print-type-size variant `Suspend2`: 1027 bytes +print-type-size local `.fut`: 1025 bytes +print-type-size local `..coroutine_field4`: 1 bytes, type: bool +print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes +print-type-size field `.value`: 1025 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1025 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1025 bytes +print-type-size type: `{async fn body of big_fut()}`: 1025 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size upvar `.arg`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.arg`: 1024 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes +print-type-size field `.waker`: 8 bytes +print-type-size field `.local_waker`: 8 bytes +print-type-size field `.ext`: 16 bytes +print-type-size field `._marker`: 0 bytes +print-type-size field `._marker2`: 0 bytes +print-type-size type: `std::panic::Location<'_>`: 24 bytes, alignment: 8 bytes +print-type-size field `.filename`: 16 bytes +print-type-size field `.line`: 4 bytes +print-type-size field `.col`: 4 bytes +print-type-size field `._filename`: 0 bytes +print-type-size type: `core::task::wake::ExtData<'_>`: 16 bytes, alignment: 8 bytes +print-type-size variant `Some`: 16 bytes +print-type-size field `.0`: 16 bytes +print-type-size variant `None`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size type: `std::panic::AssertUnwindSafe>`: 16 bytes, alignment: 8 bytes +print-type-size field `.0`: 16 bytes +print-type-size type: `std::ptr::NonNull`: 16 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 16 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of big_fut()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of test()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of wait()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::ptr::DynMetadata`: 8 bytes, alignment: 8 bytes +print-type-size field `._vtable_ptr`: 8 bytes +print-type-size field `._phantom`: 0 bytes +print-type-size type: `std::ptr::NonNull`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::mem::ManuallyDrop`: 1 bytes, alignment: 1 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::MaybeUninit`: 1 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::task::Poll<()>`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `{async fn body of wait()}`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::marker::PhantomData<&str>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData<*mut ()>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData &()>`: 0 bytes, alignment: 1 bytes diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.rs b/tests/ui/async-await/future-sizes/async-awaiting-fut.rs index 7113f591630d1..d646c7b8f039a 100644 --- a/tests/ui/async-await/future-sizes/async-awaiting-fut.rs +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only // FIXME(#61117): Respect debuginfo-level-tests, do not force debuginfo=0 //@ compile-flags: -C debuginfo=0 //@ compile-flags: -C panic=abort -Z print-type-sizes --crate-type lib diff --git a/tests/ui/async-await/future-sizes/future-as-arg.rs b/tests/ui/async-await/future-sizes/future-as-arg.rs index 317c17b9b3e59..3bbbadc2099df 100644 --- a/tests/ui/async-await/future-sizes/future-as-arg.rs +++ b/tests/ui/async-await/future-sizes/future-as-arg.rs @@ -8,9 +8,10 @@ async fn use_future(fut: impl std::future::Future) { } fn main() { - let actual = std::mem::size_of_val( - &use_future(use_future(use_future(use_future(use_future(test([0; 16]))))))); + let actual = std::mem::size_of_val(&use_future(use_future(use_future(use_future( + use_future(test([0; 16])), + ))))); // Not using an exact number in case it slightly changes over different commits - let expected = 550; - assert!(actual > expected, "expected: >{expected}, actual: {actual}"); + let expected = 21; + assert!(actual >= expected, "expected: >{expected}, actual: {actual}"); } diff --git a/tests/ui/async-await/future-sizes/large-arg.stdout b/tests/ui/async-await/future-sizes/large-arg.classic.stdout similarity index 83% rename from tests/ui/async-await/future-sizes/large-arg.stdout rename to tests/ui/async-await/future-sizes/large-arg.classic.stdout index e00420d1493f4..37bc7dc658191 100644 --- a/tests/ui/async-await/future-sizes/large-arg.stdout +++ b/tests/ui/async-await/future-sizes/large-arg.classic.stdout @@ -14,14 +14,13 @@ print-type-size field `.value`: 3075 bytes print-type-size type: `{async fn body of a<[u8; 1024]>()}`: 3075 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Suspend0`: 3074 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size local `.__awaitee`: 2050 bytes, type: {async fn body of b<[u8; 1024]>()} +print-type-size padding: 1024 bytes +print-type-size local `.__awaitee`: 2050 bytes, alignment: 1 bytes, type: {async fn body of b<[u8; 1024]>()} print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of b<[u8; 1024]>()}>`: 2050 bytes, alignment: 1 bytes print-type-size field `.value`: 2050 bytes print-type-size type: `std::mem::MaybeUninit<{async fn body of b<[u8; 1024]>()}>`: 2050 bytes, alignment: 1 bytes @@ -31,14 +30,13 @@ print-type-size field `.value`: 2050 bytes print-type-size type: `{async fn body of b<[u8; 1024]>()}`: 2050 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Suspend0`: 2049 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of c<[u8; 1024]>()} +print-type-size padding: 1024 bytes +print-type-size local `.__awaitee`: 1025 bytes, alignment: 1 bytes, type: {async fn body of c<[u8; 1024]>()} print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes print-type-size field `.value`: 1025 bytes print-type-size type: `std::mem::MaybeUninit<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes @@ -53,11 +51,16 @@ print-type-size variant `Pending`: 0 bytes print-type-size type: `{async fn body of c<[u8; 1024]>()}`: 1025 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes print-type-size field `.waker`: 8 bytes print-type-size field `.local_waker`: 8 bytes diff --git a/tests/ui/async-await/future-sizes/large-arg.relocate.stdout b/tests/ui/async-await/future-sizes/large-arg.relocate.stdout new file mode 100644 index 0000000000000..53465757a3d1e --- /dev/null +++ b/tests/ui/async-await/future-sizes/large-arg.relocate.stdout @@ -0,0 +1,103 @@ +print-type-size type: `{async fn body of test()}`: 1028 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Suspend0`: 1027 bytes +print-type-size local `.__awaitee`: 1027 bytes, type: {async fn body of a<[u8; 1024]>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of a<[u8; 1024]>()}>`: 1027 bytes, alignment: 1 bytes +print-type-size field `.value`: 1027 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of a<[u8; 1024]>()}>`: 1027 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1027 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1027 bytes +print-type-size type: `{async fn body of a<[u8; 1024]>()}`: 1027 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size upvar `.t`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.t`: 1024 bytes +print-type-size variant `Suspend0`: 1026 bytes +print-type-size local `.__awaitee`: 1026 bytes, type: {async fn body of b<[u8; 1024]>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of b<[u8; 1024]>()}>`: 1026 bytes, alignment: 1 bytes +print-type-size field `.value`: 1026 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of b<[u8; 1024]>()}>`: 1026 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1026 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1026 bytes +print-type-size type: `{async fn body of b<[u8; 1024]>()}`: 1026 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size upvar `.t`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.t`: 1024 bytes +print-type-size variant `Suspend0`: 1025 bytes +print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of c<[u8; 1024]>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes +print-type-size field `.value`: 1025 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1025 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1025 bytes +print-type-size type: `std::task::Poll<[u8; 1024]>`: 1025 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 1024 bytes +print-type-size field `.0`: 1024 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `{async fn body of c<[u8; 1024]>()}`: 1025 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size upvar `.t`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.t`: 1024 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes +print-type-size field `.waker`: 8 bytes +print-type-size field `.local_waker`: 8 bytes +print-type-size field `.ext`: 16 bytes +print-type-size field `._marker`: 0 bytes +print-type-size field `._marker2`: 0 bytes +print-type-size type: `std::panic::Location<'_>`: 24 bytes, alignment: 8 bytes +print-type-size field `.filename`: 16 bytes +print-type-size field `.line`: 4 bytes +print-type-size field `.col`: 4 bytes +print-type-size field `._filename`: 0 bytes +print-type-size type: `core::task::wake::ExtData<'_>`: 16 bytes, alignment: 8 bytes +print-type-size variant `Some`: 16 bytes +print-type-size field `.0`: 16 bytes +print-type-size variant `None`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size type: `std::panic::AssertUnwindSafe>`: 16 bytes, alignment: 8 bytes +print-type-size field `.0`: 16 bytes +print-type-size type: `std::ptr::NonNull`: 16 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 16 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of a<[u8; 1024]>()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of b<[u8; 1024]>()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of c<[u8; 1024]>()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of test()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::ptr::DynMetadata`: 8 bytes, alignment: 8 bytes +print-type-size field `._vtable_ptr`: 8 bytes +print-type-size field `._phantom`: 0 bytes +print-type-size type: `std::ptr::NonNull`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::task::Poll<()>`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `std::marker::PhantomData<&str>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData<*mut ()>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData &()>`: 0 bytes, alignment: 1 bytes diff --git a/tests/ui/async-await/future-sizes/large-arg.rs b/tests/ui/async-await/future-sizes/large-arg.rs index b05a2b7191512..2dd440b3f3a7f 100644 --- a/tests/ui/async-await/future-sizes/large-arg.rs +++ b/tests/ui/async-await/future-sizes/large-arg.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only // FIXME(#61117): Respect debuginfo-level-tests, do not force debuginfo=0 //@ compile-flags: -C debuginfo=0 //@ compile-flags: -C panic=abort -Z print-type-sizes --crate-type=lib diff --git a/tests/ui/async-await/issue-70818.stderr b/tests/ui/async-await/issue-70818.stderr index 07fd20cdd77ea..efc412cef098b 100644 --- a/tests/ui/async-await/issue-70818.stderr +++ b/tests/ui/async-await/issue-70818.stderr @@ -5,10 +5,10 @@ LL | async { (ty, ty1) } | ^^^^^^^^^^^^^^^^^^^ future created by async block is not `Send` | note: captured value is not `Send` - --> $DIR/issue-70818.rs:6:18 + --> $DIR/issue-70818.rs:4:27 | -LL | async { (ty, ty1) } - | ^^^ has type `U` which is not `Send` +LL | fn foo(ty: T, ty1: U) -> impl Future + Send { + | ^^^ has type `U` which is not `Send` note: required by a bound in an opaque type --> $DIR/issue-70818.rs:4:69 | @@ -26,10 +26,10 @@ LL | fn foo(ty: T, ty1: U) -> impl Future + Send { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ future created by async block is not `Send` | note: captured value is not `Send` - --> $DIR/issue-70818.rs:6:18 + --> $DIR/issue-70818.rs:4:27 | -LL | async { (ty, ty1) } - | ^^^ has type `U` which is not `Send` +LL | fn foo(ty: T, ty1: U) -> impl Future + Send { + | ^^^ has type `U` which is not `Send` help: consider restricting type parameter `U` with trait `Send` | LL | fn foo(ty: T, ty1: U) -> impl Future + Send { diff --git a/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs b/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs index f6c9fdd6d6806..63adb5a725172 100644 --- a/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs +++ b/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs @@ -9,7 +9,7 @@ pub async fn async_fn(x: &mut i32) -> &i32 { y } -pub fn async_closure(x: &mut i32) -> impl Future { +pub fn async_closure(x: &mut i32) -> impl Future { (async move || { //~^ ERROR lifetime may not live long enough //~| ERROR temporary value dropped while borrowed @@ -19,7 +19,7 @@ pub fn async_closure(x: &mut i32) -> impl Future { })() } -pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { +pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { (async move || -> &i32 { //~^ ERROR lifetime may not live long enough //~| ERROR temporary value dropped while borrowed @@ -29,7 +29,7 @@ pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future impl Future { +pub fn async_block(x: &mut i32) -> impl Future { async move { let y = &*x; *x += 1; //~ ERROR cannot assign to `*x` because it is borrowed diff --git a/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr b/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr index e1f268116fc54..cb559898c2fc5 100644 --- a/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr +++ b/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr @@ -44,7 +44,7 @@ LL | | })() error[E0716]: temporary value dropped while borrowed --> $DIR/issue-74072-lifetime-name-annotations.rs:13:5 | -LL | pub fn async_closure(x: &mut i32) -> impl Future { +LL | pub fn async_closure(x: &mut i32) -> impl Future { | - let's call the lifetime of this reference `'1` LL | // (async move || { LL | || @@ -93,7 +93,7 @@ LL | | })() error[E0716]: temporary value dropped while borrowed --> $DIR/issue-74072-lifetime-name-annotations.rs:23:5 | -LL | pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { +LL | pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { | - let's call the lifetime of this reference `'1` LL | // (async move || -> &i32 { LL | || diff --git a/tests/ui/async-await/issue-86507.rs b/tests/ui/async-await/issue-86507.rs index 484122a1ddcfd..6dbebc9328677 100644 --- a/tests/ui/async-await/issue-86507.rs +++ b/tests/ui/async-await/issue-86507.rs @@ -12,9 +12,14 @@ trait Foo { impl Foo for () { fn bar<'me, 'async_trait, T: Send>(x: &'me T) + //~^ NOTE captured value is not `Send` + //~| NOTE has type `&T` which is not `Send` -> Pin + Send + 'async_trait>> where 'me:'async_trait { - Box::pin( //~ ERROR future cannot be sent between threads safely + Box::pin( + //~^ ERROR future cannot be sent between threads safely + //~| NOTE future created by async block is not `Send` + //~| NOTE required for the cast from async move { let x = x; } diff --git a/tests/ui/async-await/issue-86507.stderr b/tests/ui/async-await/issue-86507.stderr index c71801dcfc854..32b9f795ee79d 100644 --- a/tests/ui/async-await/issue-86507.stderr +++ b/tests/ui/async-await/issue-86507.stderr @@ -1,19 +1,17 @@ error: future cannot be sent between threads safely - --> $DIR/issue-86507.rs:17:13 + --> $DIR/issue-86507.rs:19:13 | LL | / Box::pin( -LL | | async move { -LL | | let x = x; -LL | | } +... | LL | | ) | |_____________^ future created by async block is not `Send` | note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> $DIR/issue-86507.rs:19:29 + --> $DIR/issue-86507.rs:14:40 | -LL | let x = x; - | ^ has type `&T` which is not `Send`, because `T` is not `Sync` - = note: required for the cast from `Pin>` to `Pin + Send>>` +LL | fn bar<'me, 'async_trait, T: Send>(x: &'me T) + | ^ has type `&T` which is not `Send`, because `T` is not `Sync` + = note: required for the cast from `Pin>` to `Pin + Send>>` help: consider further restricting type parameter `T` with trait `Sync` | LL | fn bar<'me, 'async_trait, T: Send + std::marker::Sync>(x: &'me T) diff --git a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.rs b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.rs index 9317ab5cd2756..d5e849b77e812 100644 --- a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.rs +++ b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.rs @@ -2,12 +2,25 @@ //@ edition: 2018 //@ compile-flags: --target thumbv8m.main-none-eabi --crate-type lib //@ needs-llvm-components: arm -#![feature(cmse_nonsecure_entry, c_variadic, no_core, lang_items)] +#![feature(cmse_nonsecure_entry, rustc_attrs, c_variadic, no_core, lang_items, transparent_unions)] #![no_core] extern crate minicore; use minicore::*; +#[lang = "clone"] +pub trait Clone: Sized {} + +#[lang = "maybe_uninit"] +#[repr(transparent)] +#[rustc_pub_transparent] +pub union MaybeUninit { + uninit: (), + value: ManuallyDrop, +} +impl Copy for MaybeUninit {} +impl Clone for MaybeUninit {} + #[lang = "va_list"] struct VaList(*mut u8); diff --git a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.stderr b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.stderr index 948f8f5747b0e..060dc4129de2d 100644 --- a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.stderr +++ b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.stderr @@ -1,5 +1,5 @@ error: `...` is not supported for `extern "cmse-nonsecure-entry"` functions - --> $DIR/c-variadic.rs:14:60 + --> $DIR/c-variadic.rs:27:60 | LL | unsafe extern "cmse-nonsecure-entry" fn c_variadic(_: u32, _: ...) { | ----------------------------- ^^^^^^ @@ -9,13 +9,13 @@ LL | unsafe extern "cmse-nonsecure-entry" fn c_variadic(_: u32, _: ...) { = help: only `extern "C"` and `extern "C-unwind"` functions may have a C variable argument list error: functions cannot be both `async` and C-variadic - --> $DIR/c-variadic.rs:19:1 + --> $DIR/c-variadic.rs:32:1 | LL | async unsafe extern "cmse-nonsecure-entry" fn async_and_c_variadic(_: ...) { | ^^^^^ `async` because of this ^^^^^^ C-variadic because of this error: `...` is not supported for `extern "cmse-nonsecure-entry"` functions - --> $DIR/c-variadic.rs:19:68 + --> $DIR/c-variadic.rs:32:68 | LL | async unsafe extern "cmse-nonsecure-entry" fn async_and_c_variadic(_: ...) { | ----------------------------- ^^^^^^ diff --git a/tests/ui/coroutine/clone-impl.stderr b/tests/ui/coroutine/clone-impl.classic.stderr similarity index 58% rename from tests/ui/coroutine/clone-impl.stderr rename to tests/ui/coroutine/clone-impl.classic.stderr index f316902a42d08..73513cf8df612 100644 --- a/tests/ui/coroutine/clone-impl.stderr +++ b/tests/ui/coroutine/clone-impl.classic.stderr @@ -1,81 +1,81 @@ -error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:44:5: 44:12}` - --> $DIR/clone-impl.rs:48:5 +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` + --> $DIR/clone-impl.rs:51:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:44:5: 44:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` ... LL | check_copy(&gen_clone_0); - | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:44:5: 44:12}`, the trait `Copy` is not implemented for `Vec` + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}`, the trait `Copy` is not implemented for `Vec` | note: captured value does not implement `Copy` - --> $DIR/clone-impl.rs:46:14 + --> $DIR/clone-impl.rs:45:9 | -LL | drop(clonable_0); - | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +LL | let clonable_0: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` -error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:55:5: 55:12}` - --> $DIR/clone-impl.rs:60:5 +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` + --> $DIR/clone-impl.rs:63:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:55:5: 55:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` ... LL | check_copy(&gen_clone_1); - | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:55:5: 55:12}`, the trait `Copy` is not implemented for `Vec` + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}`, the trait `Copy` is not implemented for `Vec` | note: coroutine does not implement `Copy` as this value is used across a yield - --> $DIR/clone-impl.rs:57:9 + --> $DIR/clone-impl.rs:60:9 | LL | let v = vec!['a']; | - has type `Vec` which does not implement `Copy` LL | yield; | ^^^^^ yield occurs here, with `v` maybe used later note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` -error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:68:5: 68:12}` - --> $DIR/clone-impl.rs:74:5 +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` + --> $DIR/clone-impl.rs:77:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:68:5: 68:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` ... LL | check_copy(&gen_clone_1); - | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:68:5: 68:12}`, the trait `Copy` is not implemented for `Vec` + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}`, the trait `Copy` is not implemented for `Vec` | note: captured value does not implement `Copy` - --> $DIR/clone-impl.rs:72:14 + --> $DIR/clone-impl.rs:69:9 | -LL | drop(clonable_1); - | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +LL | let clonable_1: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` -error[E0277]: the trait bound `NonClone: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` - --> $DIR/clone-impl.rs:86:5 +error[E0277]: the trait bound `NonClone: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:89:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` ... LL | check_copy(&gen_non_clone); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}`, the trait `Copy` is not implemented for `NonClone` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Copy` is not implemented for `NonClone` | note: captured value does not implement `Copy` - --> $DIR/clone-impl.rs:84:14 + --> $DIR/clone-impl.rs:83:9 | -LL | drop(non_clonable); - | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Copy` +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Copy` note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` @@ -85,22 +85,22 @@ LL + #[derive(Copy)] LL | struct NonClone; | -error[E0277]: the trait bound `NonClone: Clone` is not satisfied in `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` - --> $DIR/clone-impl.rs:88:5 +error[E0277]: the trait bound `NonClone: Clone` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:91:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` ... LL | check_clone(&gen_non_clone); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}`, the trait `Clone` is not implemented for `NonClone` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Clone` is not implemented for `NonClone` | note: captured value does not implement `Clone` - --> $DIR/clone-impl.rs:84:14 + --> $DIR/clone-impl.rs:83:9 | -LL | drop(non_clonable); - | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Clone` +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Clone` note: required by a bound in `check_clone` - --> $DIR/clone-impl.rs:93:19 + --> $DIR/clone-impl.rs:96:19 | LL | fn check_clone(_x: &T) {} | ^^^^^ required by this bound in `check_clone` diff --git a/tests/ui/coroutine/clone-impl.relocate.stderr b/tests/ui/coroutine/clone-impl.relocate.stderr new file mode 100644 index 0000000000000..73513cf8df612 --- /dev/null +++ b/tests/ui/coroutine/clone-impl.relocate.stderr @@ -0,0 +1,115 @@ +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` + --> $DIR/clone-impl.rs:51:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` +... +LL | check_copy(&gen_clone_0); + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}`, the trait `Copy` is not implemented for `Vec` + | +note: captured value does not implement `Copy` + --> $DIR/clone-impl.rs:45:9 + | +LL | let clonable_0: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` + +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` + --> $DIR/clone-impl.rs:63:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` +... +LL | check_copy(&gen_clone_1); + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}`, the trait `Copy` is not implemented for `Vec` + | +note: coroutine does not implement `Copy` as this value is used across a yield + --> $DIR/clone-impl.rs:60:9 + | +LL | let v = vec!['a']; + | - has type `Vec` which does not implement `Copy` +LL | yield; + | ^^^^^ yield occurs here, with `v` maybe used later +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` + +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` + --> $DIR/clone-impl.rs:77:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` +... +LL | check_copy(&gen_clone_1); + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}`, the trait `Copy` is not implemented for `Vec` + | +note: captured value does not implement `Copy` + --> $DIR/clone-impl.rs:69:9 + | +LL | let clonable_1: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` + +error[E0277]: the trait bound `NonClone: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:89:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` +... +LL | check_copy(&gen_non_clone); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Copy` is not implemented for `NonClone` + | +note: captured value does not implement `Copy` + --> $DIR/clone-impl.rs:83:9 + | +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Copy` +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` +help: consider annotating `NonClone` with `#[derive(Copy)]` + | +LL + #[derive(Copy)] +LL | struct NonClone; + | + +error[E0277]: the trait bound `NonClone: Clone` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:91:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` +... +LL | check_clone(&gen_non_clone); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Clone` is not implemented for `NonClone` + | +note: captured value does not implement `Clone` + --> $DIR/clone-impl.rs:83:9 + | +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Clone` +note: required by a bound in `check_clone` + --> $DIR/clone-impl.rs:96:19 + | +LL | fn check_clone(_x: &T) {} + | ^^^^^ required by this bound in `check_clone` +help: consider annotating `NonClone` with `#[derive(Clone)]` + | +LL + #[derive(Clone)] +LL | struct NonClone; + | + +error: aborting due to 5 previous errors + +For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/coroutine/clone-impl.rs b/tests/ui/coroutine/clone-impl.rs index 9e04e256fc11e..17723b56449b3 100644 --- a/tests/ui/coroutine/clone-impl.rs +++ b/tests/ui/coroutine/clone-impl.rs @@ -1,7 +1,10 @@ // gate-test-coroutine_clone // Verifies that non-static coroutines can be cloned/copied if all their upvars and locals held // across awaits can be cloned/copied. -//@compile-flags: --diagnostic-width=300 +//@ revisions: classic relocate +//@ compile-flags: --diagnostic-width=300 +//@ [classic] compile-flags: -Zpack-coroutine-layout=no +//@ [relocate] compile-flags: -Zpack-coroutine-layout=captures-only #![feature(coroutines, coroutine_clone, stmt_expr_attributes)] diff --git a/tests/ui/coroutine/delayed-obligations-emit.next.stderr b/tests/ui/coroutine/delayed-obligations-emit.next.stderr index 3a3663398c9a7..af7dfc75963c2 100644 --- a/tests/ui/coroutine/delayed-obligations-emit.next.stderr +++ b/tests/ui/coroutine/delayed-obligations-emit.next.stderr @@ -4,6 +4,7 @@ error[E0275]: overflow evaluating the requirement `{async block@$DIR/delayed-obl LL | spawn(async { build_dependencies().await }); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | + = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`delayed_obligations_emit`) note: required by a bound in `spawn` --> $DIR/delayed-obligations-emit.rs:31:13 | diff --git a/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs b/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs index 0f9c56786da06..c862ba658aa01 100644 --- a/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs +++ b/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs @@ -1,16 +1,13 @@ #![feature(coroutines, stmt_expr_attributes)] fn foo(x: &i32) { - // In this case, a reference to `b` escapes the coroutine, but not - // because of a yield. We see that there is no yield in the scope of - // `b` and give the more generic error message. let mut a = &3; - let mut b = #[coroutine] + let b = #[coroutine] move || { yield (); let b = 5; a = &b; - //~^ ERROR borrowed data escapes outside of coroutine + //~^ ERROR: borrowed data escapes outside of coroutine }; } diff --git a/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr b/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr index 6fa7082c0b8b7..41b118d9f8f3a 100644 --- a/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr +++ b/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr @@ -1,5 +1,5 @@ error[E0521]: borrowed data escapes outside of coroutine - --> $DIR/ref-escapes-but-not-over-yield.rs:12:9 + --> $DIR/ref-escapes-but-not-over-yield.rs:9:9 | LL | let mut a = &3; | ----- `a` declared here, outside of the coroutine body diff --git a/tests/ui/coroutine/ref-upvar-not-send.rs b/tests/ui/coroutine/ref-upvar-not-send.rs index 89bb5e5495f45..a843e9ba29a64 100644 --- a/tests/ui/coroutine/ref-upvar-not-send.rs +++ b/tests/ui/coroutine/ref-upvar-not-send.rs @@ -11,21 +11,27 @@ fn assert_send(_: T) {} fn main() { let x: &*mut () = &std::ptr::null_mut(); + //~^ NOTE has type `&*mut ()` which is not `Send` + //~| NOTE captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` let y: &mut *mut () = &mut std::ptr::null_mut(); - assert_send(#[coroutine] move || { + //~^ NOTE has type `&mut *mut ()` which is not `Send` + //~| NOTE captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` + assert_send( //~^ ERROR coroutine cannot be sent between threads safely //~| NOTE coroutine is not `Send` - yield; - let _x = x; - }); - //~^^ NOTE captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - //~| NOTE has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` - assert_send(#[coroutine] move || { + #[coroutine] + move || { + yield; + let _x = x; + }, + ); + assert_send( //~^ ERROR coroutine cannot be sent between threads safely //~| NOTE coroutine is not `Send` - yield; - let _y = y; - }); - //~^^ NOTE captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` - //~| NOTE has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` + #[coroutine] + move || { + yield; + let _y = y; + }, + ); } diff --git a/tests/ui/coroutine/ref-upvar-not-send.stderr b/tests/ui/coroutine/ref-upvar-not-send.stderr index 3a5e8ec4dab0c..1197583c0cc08 100644 --- a/tests/ui/coroutine/ref-upvar-not-send.stderr +++ b/tests/ui/coroutine/ref-upvar-not-send.stderr @@ -1,20 +1,21 @@ error: coroutine cannot be sent between threads safely - --> $DIR/ref-upvar-not-send.rs:15:5 + --> $DIR/ref-upvar-not-send.rs:19:5 | -LL | / assert_send(#[coroutine] move || { +LL | / assert_send( LL | | LL | | -LL | | yield; -LL | | let _x = x; -LL | | }); - | |______^ coroutine is not `Send` +LL | | #[coroutine] +... | +LL | | }, +LL | | ); + | |_____^ coroutine is not `Send` | = help: the trait `Sync` is not implemented for `*mut ()` note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> $DIR/ref-upvar-not-send.rs:19:18 + --> $DIR/ref-upvar-not-send.rs:13:9 | -LL | let _x = x; - | ^ has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` +LL | let x: &*mut () = &std::ptr::null_mut(); + | ^ has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` note: required by a bound in `assert_send` --> $DIR/ref-upvar-not-send.rs:6:19 | @@ -22,22 +23,23 @@ LL | fn assert_send(_: T) {} | ^^^^ required by this bound in `assert_send` error: coroutine cannot be sent between threads safely - --> $DIR/ref-upvar-not-send.rs:23:5 + --> $DIR/ref-upvar-not-send.rs:28:5 | -LL | / assert_send(#[coroutine] move || { +LL | / assert_send( LL | | LL | | -LL | | yield; -LL | | let _y = y; -LL | | }); - | |______^ coroutine is not `Send` +LL | | #[coroutine] +... | +LL | | }, +LL | | ); + | |_____^ coroutine is not `Send` | - = help: within `{coroutine@$DIR/ref-upvar-not-send.rs:23:30: 23:37}`, the trait `Send` is not implemented for `*mut ()` + = help: within `{coroutine@$DIR/ref-upvar-not-send.rs:32:9: 32:16}`, the trait `Send` is not implemented for `*mut ()` note: captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` - --> $DIR/ref-upvar-not-send.rs:27:18 + --> $DIR/ref-upvar-not-send.rs:16:9 | -LL | let _y = y; - | ^ has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` +LL | let y: &mut *mut () = &mut std::ptr::null_mut(); + | ^ has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` note: required by a bound in `assert_send` --> $DIR/ref-upvar-not-send.rs:6:19 | diff --git a/tests/ui/coroutine/unsized-capture-across-yield.rs b/tests/ui/coroutine/unsized-capture-across-yield.rs index ee27ea064ec23..60c8f69bdb8f0 100644 --- a/tests/ui/coroutine/unsized-capture-across-yield.rs +++ b/tests/ui/coroutine/unsized-capture-across-yield.rs @@ -4,7 +4,8 @@ use std::ops::Coroutine; fn capture() -> impl Coroutine { - let b: [u8] = *(Box::new([]) as Box<[u8]>); //~ERROR he size for values of type `[u8]` cannot be known at compilation time + let b: [u8] = *(Box::new([]) as Box<[u8]>); + //~^ ERROR the size for values of type `[u8]` cannot be known at compilation time #[coroutine] move || { println!("{:?}", &b); diff --git a/tests/ui/lint/large_assignments/large_future.attribute.stderr b/tests/ui/lint/large_assignments/large_future.attribute.stderr index 734b7ff7ba22f..3c3a1fb3e5fb8 100644 --- a/tests/ui/lint/large_assignments/large_future.attribute.stderr +++ b/tests/ui/lint/large_assignments/large_future.attribute.stderr @@ -1,5 +1,5 @@ error: moving 10024 bytes - --> $DIR/large_future.rs:19:14 + --> $DIR/large_future.rs:21:14 | LL | let z = (x, 42); | ^ value moved from here @@ -12,7 +12,7 @@ LL | #![deny(large_assignments)] | ^^^^^^^^^^^^^^^^^ error: moving 10024 bytes - --> $DIR/large_future.rs:20:13 + --> $DIR/large_future.rs:22:13 | LL | let a = z.0; | ^^^ value moved from here diff --git a/tests/ui/lint/large_assignments/large_future.option-relocate.stderr b/tests/ui/lint/large_assignments/large_future.option-relocate.stderr new file mode 100644 index 0000000000000..3c3a1fb3e5fb8 --- /dev/null +++ b/tests/ui/lint/large_assignments/large_future.option-relocate.stderr @@ -0,0 +1,23 @@ +error: moving 10024 bytes + --> $DIR/large_future.rs:21:14 + | +LL | let z = (x, 42); + | ^ value moved from here + | + = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]` +note: the lint level is defined here + --> $DIR/large_future.rs:1:9 + | +LL | #![deny(large_assignments)] + | ^^^^^^^^^^^^^^^^^ + +error: moving 10024 bytes + --> $DIR/large_future.rs:22:13 + | +LL | let a = z.0; + | ^^^ value moved from here + | + = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]` + +error: aborting due to 2 previous errors + diff --git a/tests/ui/lint/large_assignments/large_future.option.stderr b/tests/ui/lint/large_assignments/large_future.option.stderr index 734b7ff7ba22f..3c3a1fb3e5fb8 100644 --- a/tests/ui/lint/large_assignments/large_future.option.stderr +++ b/tests/ui/lint/large_assignments/large_future.option.stderr @@ -1,5 +1,5 @@ error: moving 10024 bytes - --> $DIR/large_future.rs:19:14 + --> $DIR/large_future.rs:21:14 | LL | let z = (x, 42); | ^ value moved from here @@ -12,7 +12,7 @@ LL | #![deny(large_assignments)] | ^^^^^^^^^^^^^^^^^ error: moving 10024 bytes - --> $DIR/large_future.rs:20:13 + --> $DIR/large_future.rs:22:13 | LL | let a = z.0; | ^^^ value moved from here diff --git a/tests/ui/lint/large_assignments/large_future.rs b/tests/ui/lint/large_assignments/large_future.rs index 28c358bdbf086..43ae254663be9 100644 --- a/tests/ui/lint/large_assignments/large_future.rs +++ b/tests/ui/lint/large_assignments/large_future.rs @@ -3,8 +3,10 @@ #![cfg_attr(attribute, move_size_limit = "1000")] //@ build-fail //@ only-64bit -//@ revisions: attribute option -//@ [option]compile-flags: -Zmove-size-limit=1000 +//@ revisions: attribute option option-relocate +//@ compile-flags: -Zpack-coroutine-layout=no +//@ [option] compile-flags: -Zmove-size-limit=1000 +//@ [option-relocate] compile-flags: -Zmove-size-limit=1000 -Zpack-coroutine-layout=captures-only //@ edition:2018 //@ compile-flags: -Zmir-opt-level=0 diff --git a/tests/ui/mir/lint/storage-live.stderr b/tests/ui/mir/lint/storage-live.stderr index 50df9ae061fcc..fabf7f33745f0 100644 --- a/tests/ui/mir/lint/storage-live.stderr +++ b/tests/ui/mir/lint/storage-live.stderr @@ -1,4 +1,4 @@ -error: internal compiler error: broken MIR in Item(DefId(0:8 ~ storage_live[HASH]::multiple_storage)) (after pass CheckForceInline) at bb0[1]: +error: internal compiler error: broken MIR in Item(DefId(0:8 ~ storage_live[HASH]::multiple_storage)) (after pass CheckForceInline 1-1-000) at bb0[1]: StorageLive(_1) which already has storage here --> $DIR/storage-live.rs:21:13 | diff --git a/tests/ui/print_type_sizes/async.stdout b/tests/ui/print_type_sizes/async.classic.stdout similarity index 92% rename from tests/ui/print_type_sizes/async.stdout rename to tests/ui/print_type_sizes/async.classic.stdout index d3d6b6471c6ef..dcc0585a69576 100644 --- a/tests/ui/print_type_sizes/async.stdout +++ b/tests/ui/print_type_sizes/async.classic.stdout @@ -1,15 +1,14 @@ print-type-size type: `{async fn body of test()}`: 16386 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes +print-type-size local `.arg`: 8192 bytes +print-type-size upvar `.arg`: 8192 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Suspend0`: 16385 bytes -print-type-size upvar `.arg`: 8192 bytes +print-type-size padding: 8192 bytes +print-type-size local `.__awaitee`: 1 bytes, alignment: 1 bytes, type: {async fn body of wait()} print-type-size local `.arg`: 8192 bytes -print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} print-type-size variant `Returned`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes print-type-size variant `Panicked`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes print-type-size field `.value`: 8192 bytes print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes diff --git a/tests/ui/print_type_sizes/async.relocate.stdout b/tests/ui/print_type_sizes/async.relocate.stdout new file mode 100644 index 0000000000000..c6a7d8723e5c8 --- /dev/null +++ b/tests/ui/print_type_sizes/async.relocate.stdout @@ -0,0 +1,65 @@ +print-type-size type: `{async fn body of test()}`: 8194 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 8192 bytes +print-type-size upvar `.arg`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.arg`: 8192 bytes +print-type-size variant `Suspend0`: 8193 bytes +print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} +print-type-size local `.arg`: 8192 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes +print-type-size field `.waker`: 8 bytes +print-type-size field `.local_waker`: 8 bytes +print-type-size field `.ext`: 16 bytes +print-type-size field `._marker`: 0 bytes +print-type-size field `._marker2`: 0 bytes +print-type-size type: `std::panic::Location<'_>`: 24 bytes, alignment: 8 bytes +print-type-size field `.filename`: 16 bytes +print-type-size field `.line`: 4 bytes +print-type-size field `.col`: 4 bytes +print-type-size field `._filename`: 0 bytes +print-type-size type: `core::task::wake::ExtData<'_>`: 16 bytes, alignment: 8 bytes +print-type-size variant `Some`: 16 bytes +print-type-size field `.0`: 16 bytes +print-type-size variant `None`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size type: `std::panic::AssertUnwindSafe>`: 16 bytes, alignment: 8 bytes +print-type-size field `.0`: 16 bytes +print-type-size type: `std::ptr::NonNull`: 16 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 16 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of test()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of wait()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::ptr::DynMetadata`: 8 bytes, alignment: 8 bytes +print-type-size field `._vtable_ptr`: 8 bytes +print-type-size field `._phantom`: 0 bytes +print-type-size type: `std::ptr::NonNull`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::task::Poll<()>`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `{async fn body of wait()}`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::marker::PhantomData<&str>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData<*mut ()>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData &()>`: 0 bytes, alignment: 1 bytes diff --git a/tests/ui/print_type_sizes/async.rs b/tests/ui/print_type_sizes/async.rs index b6ec88426345b..4020586b64f04 100644 --- a/tests/ui/print_type_sizes/async.rs +++ b/tests/ui/print_type_sizes/async.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only // FIXME(#61117): Respect debuginfo-level-tests, do not force debuginfo=0 //@ compile-flags: -C debuginfo=0 //@ compile-flags: -C panic=abort -Z print-type-sizes --crate-type lib diff --git a/tests/ui/print_type_sizes/coroutine.classic.stdout b/tests/ui/print_type_sizes/coroutine.classic.stdout new file mode 100644 index 0000000000000..d4e9019bb6cec --- /dev/null +++ b/tests/ui/print_type_sizes/coroutine.classic.stdout @@ -0,0 +1,14 @@ +print-type-size type: `{coroutine@$DIR/coroutine.rs:14:5: 14:14}`: 8193 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 8192 bytes +print-type-size local `.array`: 8192 bytes +print-type-size upvar `.array`: 8192 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 8192 bytes +print-type-size variant `Returned`: 8192 bytes +print-type-size variant `Panicked`: 8192 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes diff --git a/tests/ui/print_type_sizes/coroutine.relocate.stdout b/tests/ui/print_type_sizes/coroutine.relocate.stdout new file mode 100644 index 0000000000000..79f5d093dc3ad --- /dev/null +++ b/tests/ui/print_type_sizes/coroutine.relocate.stdout @@ -0,0 +1,15 @@ +print-type-size type: `{coroutine@$DIR/coroutine.rs:14:5: 14:14}`: 8193 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 8192 bytes +print-type-size upvar `.array`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.array`: 8192 bytes +print-type-size variant `Suspend0`: 8192 bytes +print-type-size local `.array`: 8192 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes diff --git a/tests/ui/print_type_sizes/coroutine.rs b/tests/ui/print_type_sizes/coroutine.rs index 1533578878944..407d0873c015d 100644 --- a/tests/ui/print_type_sizes/coroutine.rs +++ b/tests/ui/print_type_sizes/coroutine.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ compile-flags: -Z print-type-sizes --crate-type=lib //@ build-pass //@ ignore-pass diff --git a/tests/ui/print_type_sizes/coroutine.stdout b/tests/ui/print_type_sizes/coroutine.stdout deleted file mode 100644 index 339bbddfc2a93..0000000000000 --- a/tests/ui/print_type_sizes/coroutine.stdout +++ /dev/null @@ -1,10 +0,0 @@ -print-type-size type: `{coroutine@$DIR/coroutine.rs:11:5: 11:14}`: 8193 bytes, alignment: 1 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `Unresumed`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Suspend0`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Returned`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Panicked`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes diff --git a/tests/ui/rustc_public-ir-print/async-closure.stdout b/tests/ui/rustc_public-ir-print/async-closure.classic.stdout similarity index 70% rename from tests/ui/rustc_public-ir-print/async-closure.stdout rename to tests/ui/rustc_public-ir-print/async-closure.classic.stdout index 3ec816b657f45..ad702cd1cee66 100644 --- a/tests/ui/rustc_public-ir-print/async-closure.stdout +++ b/tests/ui/rustc_public-ir-print/async-closure.classic.stdout @@ -3,7 +3,7 @@ fn foo() -> () { let mut _0: (); let _1: i32; - let _2: {async closure@$DIR/async-closure.rs:9:13: 9:21}; + let _2: {async closure@$DIR/async-closure.rs:12:13: 12:21}; let mut _3: &i32; debug y => _1; debug x => _2; @@ -13,7 +13,7 @@ fn foo() -> () { StorageLive(_2); StorageLive(_3); _3 = &_1; - _2 = {coroutine-closure@$DIR/async-closure.rs:9:13: 9:21}(move _3); + _2 = {coroutine-closure@$DIR/async-closure.rs:12:13: 12:21}(move _3); StorageDead(_3); _0 = (); StorageDead(_2); @@ -21,8 +21,8 @@ fn foo() -> () { return; } } -fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:9:13: 9:21}) -> {async closure body@$DIR/async-closure.rs:9:22: 11:6} { - let mut _0: {async closure body@$DIR/async-closure.rs:9:22: 11:6}; +fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:12:13: 12:21}) -> {async closure body@$DIR/async-closure.rs:12:22: 14:6} { + let mut _0: {async closure body@$DIR/async-closure.rs:12:22: 14:6}; let mut _2: &i32; let mut _3: &i32; debug y => (*((*_1).0: &i32)); @@ -30,29 +30,29 @@ fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:9:13: 9:21}) -> {a StorageLive(_2); _3 = ((*_1).0: &i32); _2 = &(*_3); - _0 = {coroutine@$DIR/async-closure.rs:9:22: 11:6}(move _2); + _0 = {coroutine@$DIR/async-closure.rs:12:22: 14:6}(move _2); StorageDead(_2); return; } } -fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}>, _2: &mut Context<'_>) -> Poll<()> { +fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { let mut _0: Poll<()>; let _3: i32; let mut _4: &i32; let mut _5: (); let mut _6: u32; - let mut _7: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; + let mut _7: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; debug _task_context => _2; - debug y => (*((*_7).0: &i32)); + debug y => (*(((*_7) as variant#0).0: &i32)); debug y => _3; bb0: { - _7 = (_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}); + _7 = (_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}); _6 = discriminant((*_7)); switchInt(move _6) -> [0: bb1, 1: bb2, otherwise: bb3]; } bb1: { StorageLive(_3); - _4 = ((*_7).0: &i32); + _4 = (((*_7) as variant#0).0: &i32); _3 = (*_4); _5 = (); StorageDead(_3); @@ -67,24 +67,24 @@ fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-clo unreachable; } } -fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}>, _2: &mut Context<'_>) -> Poll<()> { +fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { let mut _0: Poll<()>; let _3: i32; let mut _4: &i32; let mut _5: (); let mut _6: u32; - let mut _7: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; + let mut _7: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; debug _task_context => _2; - debug y => (*((*_7).0: &i32)); + debug y => (*(((*_7) as variant#0).0: &i32)); debug y => _3; bb0: { - _7 = (_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}); + _7 = (_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}); _6 = discriminant((*_7)); switchInt(move _6) -> [0: bb1, 1: bb2, otherwise: bb3]; } bb1: { StorageLive(_3); - _4 = ((*_7).0: &i32); + _4 = (((*_7) as variant#0).0: &i32); _3 = (*_4); _5 = (); StorageDead(_3); diff --git a/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout b/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout new file mode 100644 index 0000000000000..d1fe068b9c253 --- /dev/null +++ b/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout @@ -0,0 +1,119 @@ +// WARNING: This is highly experimental output it's intended for rustc_public developers only. +// If you find a bug or want to improve the output open a issue at https://github.com/rust-lang/project-stable-mir. +fn foo() -> () { + let mut _0: (); + let _1: i32; + let _2: {async closure@$DIR/async-closure.rs:12:13: 12:21}; + let mut _3: &i32; + debug y => _1; + debug x => _2; + bb0: { + StorageLive(_1); + _1 = 0_i32; + StorageLive(_2); + StorageLive(_3); + _3 = &_1; + _2 = {coroutine-closure@$DIR/async-closure.rs:12:13: 12:21}(move _3); + StorageDead(_3); + _0 = (); + StorageDead(_2); + StorageDead(_1); + return; + } +} +fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:12:13: 12:21}) -> {async closure body@$DIR/async-closure.rs:12:22: 14:6} { + let mut _0: {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _2: &i32; + let mut _3: &i32; + debug y => (*((*_1).0: &i32)); + bb0: { + StorageLive(_2); + _3 = ((*_1).0: &i32); + _2 = &(*_3); + _0 = {coroutine@$DIR/async-closure.rs:12:22: 14:6}(move _2); + StorageDead(_2); + return; + } +} +fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { + let mut _0: Poll<()>; + let _3: i32; + let mut _4: &i32; + let mut _5: &i32; + let mut _6: &i32; + let mut _7: (); + let mut _8: u32; + let mut _9: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + debug _task_context => _2; + debug y => (*_5); + debug y => _3; + debug y => _5; + bb0: { + _9 = (_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}); + _8 = discriminant((*_9)); + switchInt(move _8) -> [0: bb1, 1: bb2, otherwise: bb3]; + } + bb1: { + StorageLive(_5); + StorageLive(_6); + _6 = move (((*_9) as variant#0).0: &i32); + _5 = move _6; + StorageDead(_6); + StorageLive(_3); + _4 = _5; + _3 = (*_4); + _7 = (); + StorageDead(_3); + StorageDead(_5); + _0 = std::task::Poll::Ready(move _7); + discriminant((*_9)) = 1; + return; + } + bb2: { + assert(false, `async fn` resumed after completion) -> [success: bb2, unwind unreachable]; + } + bb3: { + unreachable; + } +} +fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { + let mut _0: Poll<()>; + let _3: i32; + let mut _4: &i32; + let mut _5: &i32; + let mut _6: &i32; + let mut _7: (); + let mut _8: u32; + let mut _9: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + debug _task_context => _2; + debug y => (*_5); + debug y => _3; + debug y => _5; + bb0: { + _9 = (_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}); + _8 = discriminant((*_9)); + switchInt(move _8) -> [0: bb1, 1: bb2, otherwise: bb3]; + } + bb1: { + StorageLive(_5); + StorageLive(_6); + _6 = move (((*_9) as variant#0).0: &i32); + _5 = move _6; + StorageDead(_6); + StorageLive(_3); + _4 = _5; + _3 = (*_4); + _7 = (); + StorageDead(_3); + StorageDead(_5); + _0 = std::task::Poll::Ready(move _7); + discriminant((*_9)) = 1; + return; + } + bb2: { + assert(false, `async fn` resumed after completion) -> [success: bb2, unwind unreachable]; + } + bb3: { + unreachable; + } +} diff --git a/tests/ui/rustc_public-ir-print/async-closure.rs b/tests/ui/rustc_public-ir-print/async-closure.rs index bd8c7e888a37c..4457468752b2e 100644 --- a/tests/ui/rustc_public-ir-print/async-closure.rs +++ b/tests/ui/rustc_public-ir-print/async-closure.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ compile-flags: -Z unpretty=stable-mir --crate-type lib -C panic=abort -Zmir-opt-level=0 //@ check-pass //@ only-64bit