From ea49d7c806185b1ee1421b49313e854a356fd2e5 Mon Sep 17 00:00:00 2001 From: Xiangfei Ding Date: Sun, 26 Oct 2025 21:57:41 +0000 Subject: [PATCH 1/2] Stop using prefix_tys This function hints at an early commitment to coroutine memory layout. We should not give promises on how upvars are allocated. Signed-off-by: Xiangfei Ding --- compiler/rustc_borrowck/src/type_check/mod.rs | 21 +++++------ .../src/debuginfo/metadata.rs | 6 ++-- .../src/debuginfo/metadata/enums/mod.rs | 35 ++----------------- compiler/rustc_middle/src/mir/statement.rs | 14 ++++---- compiler/rustc_middle/src/ty/layout.rs | 5 +-- compiler/rustc_middle/src/ty/sty.rs | 7 ---- compiler/rustc_mir_transform/src/validate.rs | 13 +++---- compiler/rustc_ty_utils/src/layout.rs | 2 +- 8 files changed, 32 insertions(+), 71 deletions(-) diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index d1d0bff8fe06b..a6976e24af12c 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -25,8 +25,8 @@ use rustc_middle::traits::query::NoSolution; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::cast::CastTy; use rustc_middle::ty::{ - self, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, CoroutineArgsExt, - GenericArgsRef, Ty, TyCtxt, TypeVisitableExt, UserArgs, UserTypeAnnotationIndex, fold_regions, + self, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, GenericArgsRef, Ty, TyCtxt, + TypeVisitableExt, UserArgs, UserTypeAnnotationIndex, fold_regions, }; use rustc_middle::{bug, span_bug}; use rustc_mir_dataflow::move_paths::MoveData; @@ -2165,14 +2165,15 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { } } AggregateKind::Coroutine(_, args) => { - // It doesn't make sense to look at a field beyond the prefix; - // these require a variant index, and are not initialized in - // aggregate rvalues. - match args.as_coroutine().prefix_tys().get(field_index.as_usize()) { - Some(ty) => Ok(*ty), - None => Err(FieldAccessError::OutOfRange { - field_count: args.as_coroutine().prefix_tys().len(), - }), + // It doesn't make sense to look at a field beyond the captured + // upvars. + // Otherwise it require a variant index, and are not initialized + // in aggregate rvalues. + let upvar_tys = &args.as_coroutine().upvar_tys(); + if let Some(ty) = upvar_tys.get(field_index.as_usize()) { + Ok(*ty) + } else { + Err(FieldAccessError::OutOfRange { field_count: upvar_tys.len() }) } } AggregateKind::CoroutineClosure(_, args) => { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 5535ebe65859e..2bd8633b63852 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -15,9 +15,7 @@ use rustc_middle::bug; use rustc_middle::ty::layout::{ HasTypingEnv, LayoutOf, TyAndLayout, WIDE_PTR_ADDR, WIDE_PTR_EXTRA, }; -use rustc_middle::ty::{ - self, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility, -}; +use rustc_middle::ty::{self, AdtKind, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility}; use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::{ DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, Span, Symbol, hygiene, @@ -1152,7 +1150,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( closure_or_coroutine_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() { - ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().prefix_tys()), + ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().upvar_tys()), ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), ty::CoroutineClosure(def_id, args) => (def_id, args.as_coroutine_closure().upvar_tys()), _ => { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index caff358607974..81425c71ac645 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -5,12 +5,11 @@ use rustc_codegen_ssa::debuginfo::type_names::{compute_debuginfo_type_name, cpp_ use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_hir::def::CtorKind; -use rustc_index::IndexSlice; use rustc_middle::bug; use rustc_middle::mir::CoroutineLayout; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef}; -use rustc_span::{Span, Symbol}; +use rustc_span::Span; use super::type_map::{DINodeCreationResult, UniqueTypeId}; use super::{SmallVec, size_and_align_of}; @@ -287,7 +286,6 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( coroutine_type_and_layout: TyAndLayout<'tcx>, coroutine_type_di_node: &'ll DIType, coroutine_layout: &CoroutineLayout<'tcx>, - common_upvar_names: &IndexSlice, ) -> &'ll DIType { let variant_name = CoroutineArgs::variant_name(variant_index); let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( @@ -298,11 +296,6 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( let variant_layout = coroutine_type_and_layout.for_variant(cx, variant_index); - let coroutine_args = match coroutine_type_and_layout.ty.kind() { - ty::Coroutine(_, args) => args.as_coroutine(), - _ => unreachable!(), - }; - type_map::build_type_with_children( cx, type_map::stub( @@ -317,7 +310,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( ), |cx, variant_struct_type_di_node| { // Fields that just belong to this variant/state - let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) + (0..variant_layout.fields.count()) .map(|field_index| { let coroutine_saved_local = coroutine_layout.variant_fields[variant_index] [FieldIdx::from_usize(field_index)]; @@ -340,29 +333,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( None, ) }) - .collect(); - - // Fields that are common to all states - let common_fields: SmallVec<_> = coroutine_args - .prefix_tys() - .iter() - .zip(common_upvar_names) - .enumerate() - .map(|(index, (upvar_ty, upvar_name))| { - build_field_di_node( - cx, - variant_struct_type_di_node, - upvar_name.as_str(), - cx.layout_of(upvar_ty), - coroutine_type_and_layout.fields.offset(index), - DIFlags::FlagZero, - type_di_node(cx, upvar_ty), - None, - ) - }) - .collect(); - - state_specific_fields.into_iter().chain(common_fields).collect() + .collect() }, |cx| build_generic_type_param_di_nodes(cx, coroutine_type_and_layout.ty), ) diff --git a/compiler/rustc_middle/src/mir/statement.rs b/compiler/rustc_middle/src/mir/statement.rs index 3cbb4b70b062a..7d69a31fcf5ee 100644 --- a/compiler/rustc_middle/src/mir/statement.rs +++ b/compiler/rustc_middle/src/mir/statement.rs @@ -156,13 +156,13 @@ impl<'tcx> PlaceTy<'tcx> { .get(f.index()) .copied() .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")), - // Only prefix fields (upvars and current state) are - // accessible without a variant index. - ty::Coroutine(_, args) => { - args.as_coroutine().prefix_tys().get(f.index()).copied().unwrap_or_else(|| { - bug!("field {f:?} out of range of prefixes for {self_ty}") - }) - } + // Only upvars are accessible without a variant index. + ty::Coroutine(_, args) => args + .as_coroutine() + .upvar_tys() + .get(f.index()) + .copied() + .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")), ty::Tuple(tys) => tys .get(f.index()) .copied() diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index eefb913a33fa4..4072bb3342495 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -956,9 +956,10 @@ where ), Variants::Multiple { tag, tag_field, .. } => { if FieldIdx::from_usize(i) == tag_field { - return TyMaybeWithLayout::TyAndLayout(tag_layout(tag)); + TyMaybeWithLayout::TyAndLayout(tag_layout(tag)) + } else { + TyMaybeWithLayout::Ty(args.as_coroutine().upvar_tys()[i]) } - TyMaybeWithLayout::Ty(args.as_coroutine().prefix_tys()[i]) } }, diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index ad8b1fd0693b0..a225140ad0466 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -150,13 +150,6 @@ impl<'tcx> ty::CoroutineArgs> { }) }) } - - /// This is the types of the fields of a coroutine which are not stored in a - /// variant. - #[inline] - fn prefix_tys(self) -> &'tcx List> { - self.upvar_tys() - } } #[derive(Debug, Copy, Clone, HashStable, TypeFoldable, TypeVisitable)] diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 5a9018a62c574..c055b31bb14b7 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -14,7 +14,7 @@ use rustc_middle::mir::*; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{ - self, CoroutineArgsExt, InstanceKind, ScalarInt, Ty, TyCtxt, TypeVisitableExt, Upcast, Variance, + self, InstanceKind, ScalarInt, Ty, TyCtxt, TypeVisitableExt, Upcast, Variance, }; use rustc_middle::{bug, span_bug}; use rustc_mir_dataflow::debuginfo::debuginfo_locals; @@ -786,14 +786,11 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { }; ty::EarlyBinder::bind(f_ty.ty).instantiate(self.tcx, args) - } else { - let Some(&f_ty) = args.as_coroutine().prefix_tys().get(f.index()) - else { - fail_out_of_bounds(self, location); - return; - }; - + } else if let Some(&f_ty) = args.as_coroutine().upvar_tys().get(f.index()) { f_ty + } else { + fail_out_of_bounds(self, location); + return; }; check_equal(self, location, f_ty); diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index b3c0953b8e147..ac284653108cd 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -528,7 +528,7 @@ fn layout_of_uncached<'tcx>( let prefix_layouts = args .as_coroutine() - .prefix_tys() + .upvar_tys() .iter() .map(|ty| cx.layout_of(ty)) .try_collect::>()?; From aa9bd749f705d2a49a25b7dbed8a9e357ab42a74 Mon Sep 17 00:00:00 2001 From: Xiangfei Ding Date: Sun, 26 Oct 2025 23:40:46 +0000 Subject: [PATCH 2/2] relocate upvars with saved locals for analysis ... and treat coroutine upvar captures as saved locals as well. This allows the liveness analysis to determine which captures are truly saved across a yield point and which are initially used but discarded at first yield points. In the event that upvar captures are promoted, most certainly because a coroutine suspends at least once, the slots in the promotion prefix shall be reused. This means that the copies emitted in the upvar relocation MIR pass will eventually elided and eliminated in the codegen phase, hence no additional runtime cost is realised. Additional MIR dumps are inserted so that it is easier to inspect the bodies of async closures, including those that captures the state by-value. Debug information is updated to point at the correct location for upvars in borrow checking errors and final debuginfo. A language change that this patch enables is now actually reverted, so that lifetimes on relocated upvars are invariant with the upvars outside of the coroutine body. We are deferring the language change to a later discussion. Co-authored-by: Dario Nieuwenhuis Signed-off-by: Xiangfei Ding --- Cargo.lock | 1 + compiler/rustc_abi/Cargo.toml | 1 + compiler/rustc_abi/src/layout.rs | 9 +- compiler/rustc_abi/src/layout/coroutine.rs | 150 ++++-- compiler/rustc_abi/src/lib.rs | 4 +- compiler/rustc_borrowck/src/lib.rs | 2 +- compiler/rustc_borrowck/src/path_utils.rs | 4 +- compiler/rustc_codegen_cranelift/src/base.rs | 3 + .../src/debuginfo/metadata/enums/cpp_like.rs | 2 - .../src/debuginfo/metadata/enums/native.rs | 4 - compiler/rustc_codegen_ssa/src/mir/operand.rs | 9 +- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 3 + .../rustc_const_eval/src/interpret/step.rs | 3 + compiler/rustc_index/src/vec.rs | 35 ++ compiler/rustc_middle/src/mir/query.rs | 26 ++ compiler/rustc_middle/src/ty/instance.rs | 7 +- compiler/rustc_middle/src/ty/mod.rs | 72 ++- compiler/rustc_mir_transform/src/coroutine.rs | 306 +++++++++--- .../src/coroutine/by_move_body.rs | 64 ++- .../rustc_mir_transform/src/coroutine/drop.rs | 18 +- .../src/coroutine/relocate_upvars.rs | 436 ++++++++++++++++++ compiler/rustc_mir_transform/src/lib.rs | 2 +- .../rustc_mir_transform/src/pass_manager.rs | 13 +- compiler/rustc_mir_transform/src/shim.rs | 6 +- .../src/shim/async_destructor_ctor.rs | 62 ++- compiler/rustc_mir_transform/src/simplify.rs | 8 +- compiler/rustc_session/src/config.rs | 21 +- compiler/rustc_session/src/options.rs | 15 + .../src/error_reporting/traits/suggestions.rs | 21 +- .../src/traits/query/dropck_outlives.rs | 11 +- .../rustc_traits/src/coroutine_witnesses.rs | 26 +- compiler/rustc_ty_utils/src/layout.rs | 31 +- .../tests/ui/explicit_write_in_test.stderr | 0 .../clippy/tests/ui/future_not_send.stderr | 4 +- .../clippy/tests/ui/needless_lifetimes.fixed | 1 + .../clippy/tests/ui/needless_lifetimes.rs | 1 + .../clippy/tests/ui/needless_lifetimes.stderr | 84 ++-- src/tools/miri/tests/pass/async-drop.rs | 8 +- .../pass/async-drop.stackrelocate.stdout | 23 + .../tests/pass/async-drop.treerelocate.stdout | 23 + tests/codegen-llvm/async-fn-debug.rs | 10 +- tests/debuginfo/coroutine-objects.rs | 12 +- .../debuginfo/coroutine-objects_relocated.rs | 90 ++++ ...#0}.coroutine_drop_async.0.panic-abort.mir | 4 +- ...0}.coroutine_drop_async.0.panic-unwind.mir | 4 +- ...await.a-{closure#0}.coroutine_resume.0.mir | 1 + ...await.b-{closure#0}.coroutine_resume.0.mir | 21 +- ....main-{closure#0}.StateTransform.after.mir | 1 + ....main-{closure#1}.StateTransform.after.mir | 1 + ...closure#0}.RelocateUpvars.panic-abort.diff | 91 ++++ ...losure#0}.RelocateUpvars.panic-unwind.diff | 104 +++++ tests/mir-opt/coroutine_relocate_upvars.rs | 21 + ...ny.main-{closure#0}.coroutine_resume.0.mir | 1 + ...y.run2-{closure#0}.Inline.panic-abort.diff | 2 +- ....run2-{closure#0}.Inline.panic-unwind.diff | 2 +- ... => async-drop-initial.classic.run.stdout} | 1 + .../async-drop-initial.relocate.run.stdout | 23 + .../async-drop/async-drop-initial.rs | 59 +-- ...dout => async-awaiting-fut.classic.stdout} | 27 +- .../async-awaiting-fut.relocate.stdout | 111 +++++ .../future-sizes/async-awaiting-fut.rs | 3 + .../async-await/future-sizes/future-as-arg.rs | 9 +- ...ge-arg.stdout => large-arg.classic.stdout} | 29 +- .../future-sizes/large-arg.relocate.stdout | 103 +++++ .../ui/async-await/future-sizes/large-arg.rs | 3 + tests/ui/async-await/issue-70818.stderr | 12 +- .../issue-74072-lifetime-name-annotations.rs | 6 +- ...sue-74072-lifetime-name-annotations.stderr | 4 +- tests/ui/async-await/issue-86507.rs | 7 +- tests/ui/async-await/issue-86507.stderr | 14 +- .../cmse-nonsecure-entry/c-variadic.rs | 15 +- .../cmse-nonsecure-entry/c-variadic.stderr | 6 +- ...-impl.stderr => clone-impl.classic.stderr} | 76 +-- tests/ui/coroutine/clone-impl.relocate.stderr | 115 +++++ tests/ui/coroutine/clone-impl.rs | 5 +- .../delayed-obligations-emit.next.stderr | 1 + .../ref-escapes-but-not-over-yield.rs | 7 +- .../ref-escapes-but-not-over-yield.stderr | 2 +- tests/ui/coroutine/ref-upvar-not-send.rs | 30 +- tests/ui/coroutine/ref-upvar-not-send.stderr | 40 +- .../coroutine/unsized-capture-across-yield.rs | 3 +- .../large_future.attribute.stderr | 4 +- .../large_future.option-relocate.stderr | 23 + .../large_future.option.stderr | 4 +- .../ui/lint/large_assignments/large_future.rs | 6 +- tests/ui/mir/lint/storage-live.stderr | 2 +- .../{async.stdout => async.classic.stdout} | 9 +- .../ui/print_type_sizes/async.relocate.stdout | 65 +++ tests/ui/print_type_sizes/async.rs | 3 + .../print_type_sizes/coroutine.classic.stdout | 14 + .../coroutine.relocate.stdout | 15 + tests/ui/print_type_sizes/coroutine.rs | 3 + tests/ui/print_type_sizes/coroutine.stdout | 10 - ...re.stdout => async-closure.classic.stdout} | 30 +- .../async-closure.relocate.stdout | 119 +++++ .../ui/rustc_public-ir-print/async-closure.rs | 3 + 96 files changed, 2415 insertions(+), 490 deletions(-) create mode 100644 compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs delete mode 100644 src/tools/clippy/tests/ui/explicit_write_in_test.stderr create mode 100644 src/tools/miri/tests/pass/async-drop.stackrelocate.stdout create mode 100644 src/tools/miri/tests/pass/async-drop.treerelocate.stdout create mode 100644 tests/debuginfo/coroutine-objects_relocated.rs create mode 100644 tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-abort.diff create mode 100644 tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-unwind.diff create mode 100644 tests/mir-opt/coroutine_relocate_upvars.rs rename tests/ui/async-await/async-drop/{async-drop-initial.run.stdout => async-drop-initial.classic.run.stdout} (94%) create mode 100644 tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout rename tests/ui/async-await/future-sizes/{async-awaiting-fut.stdout => async-awaiting-fut.classic.stdout} (88%) create mode 100644 tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout rename tests/ui/async-await/future-sizes/{large-arg.stdout => large-arg.classic.stdout} (83%) create mode 100644 tests/ui/async-await/future-sizes/large-arg.relocate.stdout rename tests/ui/coroutine/{clone-impl.stderr => clone-impl.classic.stderr} (58%) create mode 100644 tests/ui/coroutine/clone-impl.relocate.stderr create mode 100644 tests/ui/lint/large_assignments/large_future.option-relocate.stderr rename tests/ui/print_type_sizes/{async.stdout => async.classic.stdout} (92%) create mode 100644 tests/ui/print_type_sizes/async.relocate.stdout create mode 100644 tests/ui/print_type_sizes/coroutine.classic.stdout create mode 100644 tests/ui/print_type_sizes/coroutine.relocate.stdout delete mode 100644 tests/ui/print_type_sizes/coroutine.stdout rename tests/ui/rustc_public-ir-print/{async-closure.stdout => async-closure.classic.stdout} (70%) create mode 100644 tests/ui/rustc_public-ir-print/async-closure.relocate.stdout diff --git a/Cargo.lock b/Cargo.lock index 2320e33bc4bf1..ea96332ee54f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3338,6 +3338,7 @@ dependencies = [ "bitflags", "rand 0.9.2", "rand_xoshiro", + "rustc-hash 2.1.1", "rustc_data_structures", "rustc_error_messages", "rustc_hashes", diff --git a/compiler/rustc_abi/Cargo.toml b/compiler/rustc_abi/Cargo.toml index 83d96d8d04daf..af4db5844ded9 100644 --- a/compiler/rustc_abi/Cargo.toml +++ b/compiler/rustc_abi/Cargo.toml @@ -8,6 +8,7 @@ edition = "2024" bitflags = "2.4.1" rand = { version = "0.9.0", default-features = false, optional = true } rand_xoshiro = { version = "0.7.0", optional = true } +rustc-hash = "2.0.0" rustc_data_structures = { path = "../rustc_data_structures", optional = true } rustc_error_messages = { path = "../rustc_error_messages", optional = true } rustc_hashes = { path = "../rustc_hashes" } diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 14356813b7bb0..d1668fd50f445 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -3,6 +3,7 @@ use std::fmt::{self, Write}; use std::ops::{Bound, Deref}; use std::{cmp, iter}; +pub use coroutine::PackCoroutineLayout; use rustc_hashes::Hash64; use rustc_index::Idx; use rustc_index::bit_set::BitMatrix; @@ -210,17 +211,21 @@ impl LayoutCalculator { >( &self, local_layouts: &IndexSlice, - prefix_layouts: IndexVec, + relocated_upvars: &IndexSlice>, + upvar_layouts: IndexVec, variant_fields: &IndexSlice>, storage_conflicts: &BitMatrix, + pack: PackCoroutineLayout, tag_to_layout: impl Fn(Scalar) -> F, ) -> LayoutCalculatorResult { coroutine::layout( self, local_layouts, - prefix_layouts, + relocated_upvars, + upvar_layouts, variant_fields, storage_conflicts, + pack, tag_to_layout, ) } diff --git a/compiler/rustc_abi/src/layout/coroutine.rs b/compiler/rustc_abi/src/layout/coroutine.rs index 2b22276d4aed7..05140419a3efb 100644 --- a/compiler/rustc_abi/src/layout/coroutine.rs +++ b/compiler/rustc_abi/src/layout/coroutine.rs @@ -30,6 +30,17 @@ use crate::{ StructKind, TagEncoding, Variants, WrappingRange, }; +/// This option controls how coroutine saved locals are packed +/// into the coroutine state data +#[derive(Debug, Clone, Copy)] +pub enum PackCoroutineLayout { + /// The classic layout where captures are always promoted to coroutine state prefix + Classic, + /// Captures are first saved into the `UNRESUME` state and promoted + /// when they are used across more than one suspension + CapturesOnly, +} + /// Overlap eligibility and variant assignment for each CoroutineSavedLocal. #[derive(Clone, Debug, PartialEq)] enum SavedLocalEligibility { @@ -74,6 +85,7 @@ fn coroutine_saved_local_eligibility( calc: &super::LayoutCalculator, local_layouts: &IndexSlice, - mut prefix_layouts: IndexVec, + relocated_upvars: &IndexSlice>, + upvar_layouts: IndexVec, variant_fields: &IndexSlice>, storage_conflicts: &BitMatrix, + pack: PackCoroutineLayout, tag_to_layout: impl Fn(Scalar) -> F, ) -> super::LayoutCalculatorResult { use SavedLocalEligibility::*; let (ineligible_locals, assignments) = coroutine_saved_local_eligibility(local_layouts.len(), variant_fields, storage_conflicts); + debug!(?ineligible_locals); - // Build a prefix layout, including "promoting" all ineligible - // locals as part of the prefix. We compute the layout of all of - // these fields at once to get optimal packing. - let tag_index = prefix_layouts.next_index(); + // Build a prefix layout, consisting of only the state tag and, as per request, upvars + let tag_index = match pack { + PackCoroutineLayout::CapturesOnly => FieldIdx::new(0), + PackCoroutineLayout::Classic => upvar_layouts.next_index(), + }; // `variant_fields` already accounts for the reserved variants, so no need to add them. let max_discr = (variant_fields.len() - 1) as u128; @@ -168,18 +186,38 @@ pub(super) fn layout< valid_range: WrappingRange { start: 0, end: max_discr }, }; - let promoted_layouts = ineligible_locals.iter().map(|local| local_layouts[local]); - prefix_layouts.push(tag_to_layout(tag)); - prefix_layouts.extend(promoted_layouts); + let upvars_in_unresumed: rustc_hash::FxHashSet<_> = + variant_fields[VariantIdx::new(0)].iter().copied().collect(); + let promoted_layouts = ineligible_locals.iter().filter_map(|local| { + if matches!(pack, PackCoroutineLayout::Classic) && upvars_in_unresumed.contains(&local) { + // We do not need to promote upvars, they are already in the upvar region + None + } else { + Some(local_layouts[local]) + } + }); + // FIXME: when we introduce more pack scheme, we need to change the prefix layout here + let prefix_layouts: IndexVec<_, _> = match pack { + PackCoroutineLayout::Classic => { + // Classic scheme packs the states as follows + // [ .. , , ] ++ + // In addition, UNRESUME overlaps with the part + upvar_layouts.into_iter().chain([tag_to_layout(tag)]).chain(promoted_layouts).collect() + } + PackCoroutineLayout::CapturesOnly => { + [tag_to_layout(tag)].into_iter().chain(promoted_layouts).collect() + } + }; + debug!(?prefix_layouts, ?pack); let prefix = calc.univariant(&prefix_layouts, &ReprOptions::default(), StructKind::AlwaysSized)?; let (prefix_size, prefix_align) = (prefix.size, prefix.align); - // Split the prefix layout into the "outer" fields (upvars and - // discriminant) and the "promoted" fields. Promoted fields will - // get included in each variant that requested them in - // CoroutineLayout. + // Split the prefix layout into the discriminant and + // the "promoted" fields. + // Promoted fields will get included in each variant + // that requested them in CoroutineLayout. debug!("prefix = {:#?}", prefix); let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields { FieldsShape::Arbitrary { mut offsets, memory_index } => { @@ -213,24 +251,75 @@ pub(super) fn layout< _ => unreachable!(), }; + // Here we start to compute layout of each state variant let mut size = prefix.size; let mut align = prefix.align; let variants = variant_fields .iter_enumerated() .map(|(index, variant_fields)| { + // Special case: UNRESUMED overlaps with the upvar region of the prefix, + // so that moving upvars may eventually become a no-op. + let is_unresumed = index.index() == 0; + if is_unresumed && matches!(pack, PackCoroutineLayout::Classic) { + let fields = FieldsShape::Arbitrary { + offsets: (0..tag_index.index()).map(|i| outer_fields.offset(i)).collect(), + memory_index: (0..tag_index.index()) + .map(|i| { + (outer_fields.memory_index(i) + promoted_memory_index.len()) as u32 + }) + .collect(), + }; + let align = prefix.align; + let size = prefix.size; + return Ok(LayoutData { + fields, + variants: Variants::Single { index }, + backend_repr: BackendRepr::Memory { sized: true }, + largest_niche: None, + uninhabited: false, + align, + size, + max_repr_align: None, + unadjusted_abi_align: align.abi, + randomization_seed: Default::default(), + }); + } + let mut is_ineligible = IndexVec::from_elem_n(None, variant_fields.len()); + for (field, &local) in variant_fields.iter_enumerated() { + if is_unresumed { + if let Some(inner_local) = relocated_upvars[local] + && let Ineligible(Some(promoted_field)) = assignments[inner_local] + { + is_ineligible.insert(field, promoted_field); + continue; + } + } + match assignments[local] { + Assigned(v) if v == index => {} + Ineligible(Some(promoted_field)) => { + is_ineligible.insert(field, promoted_field); + } + Ineligible(None) => { + panic!("an ineligible local should have been promoted into the prefix") + } + Assigned(_) => { + panic!("an eligible local should have been assigned to exactly one variant") + } + Unassigned => { + panic!("each saved local should have been inspected at least once") + } + } + } // Only include overlap-eligible fields when we compute our variant layout. - let variant_only_tys = variant_fields - .iter() - .filter(|local| match assignments[**local] { - Unassigned => unreachable!(), - Assigned(v) if v == index => true, - Assigned(_) => unreachable!("assignment does not match variant"), - Ineligible(_) => false, + let fields: IndexVec<_, _> = variant_fields + .iter_enumerated() + .filter_map(|(field, &local)| { + if is_ineligible.contains(field) { None } else { Some(local_layouts[local]) } }) - .map(|local| local_layouts[*local]); + .collect(); let mut variant = calc.univariant( - &variant_only_tys.collect::>(), + &fields, &ReprOptions::default(), StructKind::Prefixed(prefix_size, prefix_align.abi), )?; @@ -254,19 +343,14 @@ pub(super) fn layout< IndexVec::from_elem_n(FieldIdx::new(invalid_field_idx), invalid_field_idx); let mut offsets_and_memory_index = iter::zip(offsets, memory_index); - let combined_offsets = variant_fields + let combined_offsets = is_ineligible .iter_enumerated() - .map(|(i, local)| { - let (offset, memory_index) = match assignments[*local] { - Unassigned => unreachable!(), - Assigned(_) => { - let (offset, memory_index) = offsets_and_memory_index.next().unwrap(); - (offset, promoted_memory_index.len() as u32 + memory_index) - } - Ineligible(field_idx) => { - let field_idx = field_idx.unwrap(); - (promoted_offsets[field_idx], promoted_memory_index[field_idx]) - } + .map(|(i, &is_ineligible)| { + let (offset, memory_index) = if let Some(field_idx) = is_ineligible { + (promoted_offsets[field_idx], promoted_memory_index[field_idx]) + } else { + let (offset, memory_index) = offsets_and_memory_index.next().unwrap(); + (offset, promoted_memory_index.len() as u32 + memory_index) }; combined_inverse_memory_index[memory_index] = i; offset diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index de44c8755a078..fa9d947cd9f1d 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -67,7 +67,9 @@ pub use canon_abi::{ArmCall, CanonAbi, InterruptKind, X86Call}; pub use extern_abi::CVariadicStatus; pub use extern_abi::{ExternAbi, all_names}; #[cfg(feature = "nightly")] -pub use layout::{FIRST_VARIANT, FieldIdx, Layout, TyAbiInterface, TyAndLayout, VariantIdx}; +pub use layout::{ + FIRST_VARIANT, FieldIdx, Layout, PackCoroutineLayout, TyAbiInterface, TyAndLayout, VariantIdx, +}; pub use layout::{LayoutCalculator, LayoutCalculatorError}; /// Requirements for a `StableHashingContext` to be used in this crate. diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index a57689a45b67b..a8fab1be0eca2 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -2478,7 +2478,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { // If the local may have been initialized, and it is now currently being // mutated, then it is justified to be annotated with the `mut` // keyword, since the mutation may be a possible reassignment. - if is_local_mutation_allowed != LocalMutationIsAllowed::Yes + if !matches!(is_local_mutation_allowed, LocalMutationIsAllowed::Yes) && self.is_local_ever_initialized(local, state).is_some() { self.used_mut.insert(local); diff --git a/compiler/rustc_borrowck/src/path_utils.rs b/compiler/rustc_borrowck/src/path_utils.rs index 2c94a32d369ce..6b3169223029a 100644 --- a/compiler/rustc_borrowck/src/path_utils.rs +++ b/compiler/rustc_borrowck/src/path_utils.rs @@ -3,7 +3,7 @@ use std::ops::ControlFlow; use rustc_abi::FieldIdx; use rustc_data_structures::graph::dominators::Dominators; use rustc_middle::mir::{BasicBlock, Body, Location, Place, PlaceRef, ProjectionElem}; -use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::{CapturedPlace, TyCtxt}; use tracing::debug; use crate::borrow_set::{BorrowData, BorrowSet, TwoPhaseActivation}; @@ -133,7 +133,7 @@ pub(super) fn borrow_of_local_data(place: Place<'_>) -> bool { /// of a closure type. pub(crate) fn is_upvar_field_projection<'tcx>( tcx: TyCtxt<'tcx>, - upvars: &[&rustc_middle::ty::CapturedPlace<'tcx>], + upvars: &[&CapturedPlace<'tcx>], place_ref: PlaceRef<'tcx>, body: &Body<'tcx>, ) -> Option { diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 7d50548b40262..cbef1848fefec 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -891,6 +891,9 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: let variant_dest = lval.downcast_variant(fx, variant_index); (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_def_id, _args) => { + (FIRST_VARIANT, lval.downcast_variant(fx, FIRST_VARIANT), None) + } _ => (FIRST_VARIANT, lval, None), }; if active_field_index.is_some() { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 4ecc3086e1bdf..769c34f51a04c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -721,7 +721,6 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( let coroutine_layout = cx.tcx.coroutine_layout(coroutine_def_id, coroutine_args.args).unwrap(); - let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); let variant_range = coroutine_args.variant_range(coroutine_def_id, cx.tcx); let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); @@ -761,7 +760,6 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( coroutine_type_and_layout, coroutine_type_di_node, coroutine_layout, - common_upvar_names, ); let span = coroutine_layout.variant_source_info[variant_index].span; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 1ae6e6e5eecab..44f4426c699f9 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -186,9 +186,6 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( ) }; - let common_upvar_names = - cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); - // Build variant struct types let variant_struct_type_di_nodes: SmallVec<_> = variants .indices() @@ -216,7 +213,6 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( coroutine_type_and_layout, coroutine_type_di_node, coroutine_layout, - common_upvar_names, ), source_info, } diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 88a8e2a844cbc..b187cfcc14b70 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -934,13 +934,12 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue { } impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + #[instrument(level = "debug", skip(self, bx), ret)] fn maybe_codegen_consume_direct( &mut self, bx: &mut Bx, place_ref: mir::PlaceRef<'tcx>, ) -> Option> { - debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref); - match self.locals[place_ref.local] { LocalRef::Operand(mut o) => { // We only need to handle the projections that @@ -980,13 +979,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } + #[instrument(level = "debug", skip(self, bx), ret)] pub fn codegen_consume( &mut self, bx: &mut Bx, place_ref: mir::PlaceRef<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { - debug!("codegen_consume(place_ref={:?})", place_ref); - let ty = self.monomorphized_place_ty(place_ref); let layout = bx.cx().layout_of(ty); @@ -1005,13 +1003,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.load_operand(place) } + #[instrument(level = "debug", skip(self, bx), ret)] pub fn codegen_operand( &mut self, bx: &mut Bx, operand: &mir::Operand<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { - debug!("codegen_operand(operand={:?})", operand); - match *operand { mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => { self.codegen_consume(bx, place.as_ref()) diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 640f7211dc994..88c52b8643538 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -167,6 +167,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let variant_dest = dest.project_downcast(bx, variant_index); (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_, _) => { + (FIRST_VARIANT, dest.project_downcast(bx, FIRST_VARIANT), None) + } _ => (FIRST_VARIANT, dest, None), }; if active_field_index.is_some() { diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 1766bbe92480a..fe5e87bb8ab7d 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -301,6 +301,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let variant_dest = self.project_downcast(dest, variant_index)?; (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_def_id, _args) => { + (FIRST_VARIANT, self.project_downcast(dest, FIRST_VARIANT)?, None) + } mir::AggregateKind::RawPtr(..) => { // Pointers don't have "fields" in the normal sense, so the // projection-based code below would either fail in projection diff --git a/compiler/rustc_index/src/vec.rs b/compiler/rustc_index/src/vec.rs index 13f0dda180be9..90eb212e69bf5 100644 --- a/compiler/rustc_index/src/vec.rs +++ b/compiler/rustc_index/src/vec.rs @@ -197,6 +197,11 @@ impl IndexVec { pub fn append(&mut self, other: &mut Self) { self.raw.append(&mut other.raw); } + + #[inline] + pub fn debug_map_view(&self) -> IndexSliceMapView<'_, I, T> { + IndexSliceMapView(self.as_slice()) + } } /// `IndexVec` is often used as a map, so it provides some map-like APIs. @@ -220,14 +225,44 @@ impl IndexVec> { pub fn contains(&self, index: I) -> bool { self.get(index).and_then(Option::as_ref).is_some() } + + #[inline] + pub fn debug_map_view_compact(&self) -> IndexSliceMapViewCompact<'_, I, T> { + IndexSliceMapViewCompact(self.as_slice()) + } } +pub struct IndexSliceMapView<'a, I: Idx, T>(&'a IndexSlice); +pub struct IndexSliceMapViewCompact<'a, I: Idx, T>(&'a IndexSlice>); + impl fmt::Debug for IndexVec { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&self.raw, fmt) } } +impl<'a, I: Idx, T: fmt::Debug> fmt::Debug for IndexSliceMapView<'a, I, T> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut entries = fmt.debug_map(); + for (idx, val) in self.0.iter_enumerated() { + entries.entry(&idx, val); + } + entries.finish() + } +} + +impl<'a, I: Idx, T: fmt::Debug> fmt::Debug for IndexSliceMapViewCompact<'a, I, T> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut entries = fmt.debug_map(); + for (idx, val) in self.0.iter_enumerated() { + if let Some(val) = val { + entries.entry(&idx, val); + } + } + entries.finish() + } +} + impl Deref for IndexVec { type Target = IndexSlice; diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs index 2e6c9f207e26b..eeb396fe36d0d 100644 --- a/compiler/rustc_middle/src/mir/query.rs +++ b/compiler/rustc_middle/src/mir/query.rs @@ -9,7 +9,9 @@ use rustc_hir::def_id::LocalDefId; use rustc_index::IndexVec; use rustc_index::bit_set::BitMatrix; use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable}; +use rustc_session::config::PackCoroutineLayout; use rustc_span::{Span, Symbol}; +use rustc_type_ir::data_structures::IndexMap; use super::{ConstValue, SourceInfo}; use crate::ty::{self, CoroutineArgsExt, OpaqueHiddenType, Ty}; @@ -55,6 +57,29 @@ pub struct CoroutineLayout<'tcx> { #[type_foldable(identity)] #[type_visitable(ignore)] pub storage_conflicts: BitMatrix, + + /// This map `A -> B` allows later MIR passes, error reporters + /// and layout calculator to relate saved locals `A` sourced from upvars + /// and locals `B` that upvars are moved into. + /// + /// For instance, an upvar `_1.0` is assigned saved local `_s12`, + /// see notation of [`CoroutineSavedLocal`], in the UNRESUMED state and + /// further moved into the internal saved local `_s13`. + /// This map, therefore, establishes the mapping from `_s12` to `_s13`, + /// so that their memory layout within the coroutine should be overlapped. + #[type_foldable(identity)] + #[type_visitable(ignore)] + pub relocated_upvars: IndexMap, + + /// Coroutine layout packing + #[type_foldable(identity)] + #[type_visitable(ignore)] + pub pack: PackCoroutineLayout, +} + +impl<'tcx> CoroutineLayout<'tcx> { + /// The initial state of a coroutine + pub const UNRESUMED: VariantIdx = VariantIdx::ZERO; } impl Debug for CoroutineLayout<'_> { @@ -80,6 +105,7 @@ impl Debug for CoroutineLayout<'_> { map.finish() }) .field("storage_conflicts", &self.storage_conflicts) + .field("relocated_upvars", &self.relocated_upvars) .finish() } } diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs index c27d47fcc0d8d..d0187befd0b73 100644 --- a/compiler/rustc_middle/src/ty/instance.rs +++ b/compiler/rustc_middle/src/ty/instance.rs @@ -409,15 +409,14 @@ impl<'tcx> fmt::Display for Instance<'tcx> { // async_drop_in_place::coroutine's poll function (through FutureDropPollShim proxy) fn resolve_async_drop_poll<'tcx>(mut cor_ty: Ty<'tcx>) -> Instance<'tcx> { let first_cor = cor_ty; - let ty::Coroutine(poll_def_id, proxy_args) = first_cor.kind() else { + let &ty::Coroutine(poll_def_id, proxy_args) = first_cor.kind() else { bug!(); }; - let poll_def_id = *poll_def_id; let mut child_ty = cor_ty; loop { - if let ty::Coroutine(child_def, child_args) = child_ty.kind() { + if let &ty::Coroutine(child_def, child_args) = child_ty.kind() { cor_ty = child_ty; - if *child_def == poll_def_id { + if child_def == poll_def_id { child_ty = child_args.first().unwrap().expect_ty(); continue; } else { diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 93d0c77c1daee..eeef59e3324a0 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -46,6 +46,7 @@ use rustc_macros::{ }; use rustc_query_system::ich::StableHashingContext; use rustc_serialize::{Decodable, Encodable}; +use rustc_session::config::PackCoroutineLayout; pub use rustc_session::lint::RegisteredTools; use rustc_span::hygiene::MacroKind; use rustc_span::{DUMMY_SP, ExpnId, ExpnKind, Ident, Span, Symbol, sym}; @@ -111,7 +112,7 @@ pub use self::typeck_results::{ use crate::error::{OpaqueHiddenTypeMismatch, TypeMismatchReason}; use crate::metadata::ModChild; use crate::middle::privacy::EffectiveVisibilities; -use crate::mir::{Body, CoroutineLayout, CoroutineSavedLocal, SourceInfo}; +use crate::mir::{Body, CoroutineLayout, CoroutineSavedLocal, CoroutineSavedTy, SourceInfo}; use crate::query::{IntoQueryParam, Providers}; use crate::ty; use crate::ty::codec::{TyDecoder, TyEncoder}; @@ -1872,39 +1873,68 @@ impl<'tcx> TyCtxt<'tcx> { .ok_or_else(|| self.layout_error(LayoutError::Unknown(ty()))) } + /// Construct the trivial coroutine layout for async_drop of a coroutine + /// + /// The only upvar in question is the coroutine state machine to be dropped. + #[instrument(level = "debug", ret, skip(self))] + fn build_async_drop_trivial_coroutine_layout( + self, + def_id: DefId, + upvar_ty: Ty<'tcx>, + ) -> CoroutineLayout<'tcx> { + let span = self.def_span(def_id); + let source_info = SourceInfo::outermost(span); + let mut field_tys = IndexVec::new(); + let field_names = [None].into(); + let upvar_saved_local = + field_tys.push(CoroutineSavedTy { ty: upvar_ty, source_info, ignore_for_traits: true }); + // Even minimal, the trivial coroutine has 3 states (RESERVED_VARIANTS), + // so variant_fields and variant_source_info should have 3 elements. + let mut variant_fields: IndexVec> = + iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + variant_fields[VariantIdx::ZERO].push(upvar_saved_local); + let variant_source_info: IndexVec = + iter::repeat(source_info).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + let mut storage_conflicts = BitMatrix::new(1, 1); + storage_conflicts.insert(upvar_saved_local, upvar_saved_local); + + let relocated_upvars = Default::default(); + CoroutineLayout { + field_tys, + field_names, + variant_fields, + variant_source_info, + storage_conflicts, + relocated_upvars, + pack: PackCoroutineLayout::CapturesOnly, + } + } + /// Returns layout of a coroutine. Layout might be unavailable if the /// coroutine is tainted by errors. + #[instrument(level = "debug", skip(self), ret)] pub fn coroutine_layout( self, def_id: DefId, args: GenericArgsRef<'tcx>, ) -> Result<&'tcx CoroutineLayout<'tcx>, &'tcx LayoutError<'tcx>> { if self.is_async_drop_in_place_coroutine(def_id) { - // layout of `async_drop_in_place::{closure}` in case, - // when T is a coroutine, contains this internal coroutine's ptr in upvars - // and doesn't require any locals. Here is an `empty coroutine's layout` + // Layout of `async_drop_in_place::{closure}` when T is a coroutine + // It contains exactly one upvar which is a raw pointer to this inner coroutine + // and it does not suspend. let arg_cor_ty = args.first().unwrap().expect_ty(); - if arg_cor_ty.is_coroutine() { - let span = self.def_span(def_id); - let source_info = SourceInfo::outermost(span); - // Even minimal, empty coroutine has 3 states (RESERVED_VARIANTS), - // so variant_fields and variant_source_info should have 3 elements. - let variant_fields: IndexVec> = - iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); - let variant_source_info: IndexVec = - iter::repeat(source_info).take(CoroutineArgs::RESERVED_VARIANTS).collect(); - let proxy_layout = CoroutineLayout { - field_tys: [].into(), - field_names: [].into(), - variant_fields, - variant_source_info, - storage_conflicts: BitMatrix::new(0, 0), - }; - return Ok(self.arena.alloc(proxy_layout)); + if let &ty::Coroutine(_did, _args) = arg_cor_ty.kind() { + debug!("it is a trivial coroutine"); + let upvar_ty = Ty::new_mut_ptr(self, arg_cor_ty); + Ok(self + .arena + .alloc(self.build_async_drop_trivial_coroutine_layout(def_id, upvar_ty))) } else { + debug!("it is an async drop coroutine"); self.async_drop_coroutine_layout(def_id, args) } } else { + debug!("it is an ordinary coroutine"); self.ordinary_coroutine_layout(def_id, args) } } diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 2ccd8178e667b..0d344ef2d1d86 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -52,7 +52,9 @@ mod by_move_body; mod drop; -use std::{iter, ops}; +mod relocate_upvars; + +use std::ops::Deref; pub(super) use by_move_body::coroutine_by_move_body_def_id; use drop::{ @@ -60,14 +62,16 @@ use drop::{ create_coroutine_drop_shim_proxy_async, elaborate_coroutine_drops, expand_async_drops, has_expandable_async_drops, insert_clean_drop, }; +pub(super) use relocate_upvars::RelocateUpvars; use rustc_abi::{FieldIdx, VariantIdx}; use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::unord::UnordMap; use rustc_errors::pluralize; use rustc_hir as hir; use rustc_hir::lang_items::LangItem; use rustc_hir::{CoroutineDesugaring, CoroutineKind}; use rustc_index::bit_set::{BitMatrix, DenseBitSet, GrowableBitSet}; -use rustc_index::{Idx, IndexVec, indexvec}; +use rustc_index::{Idx, IndexSlice, IndexVec, indexvec}; use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::util::Discr; @@ -82,13 +86,15 @@ use rustc_mir_dataflow::impls::{ use rustc_mir_dataflow::{ Analysis, Results, ResultsCursor, ResultsVisitor, visit_reachable_results, }; +use rustc_session::config::PackCoroutineLayout; use rustc_span::def_id::{DefId, LocalDefId}; use rustc_span::source_map::dummy_spanned; use rustc_span::symbol::sym; -use rustc_span::{DUMMY_SP, Span}; +use rustc_span::{DUMMY_SP, Span, Symbol, kw}; use rustc_trait_selection::error_reporting::InferCtxtErrorExt; use rustc_trait_selection::infer::TyCtxtInferExt as _; use rustc_trait_selection::traits::{ObligationCause, ObligationCauseCode, ObligationCtxt}; +use smallvec::{SmallVec, smallvec}; use tracing::{debug, instrument, trace}; use crate::deref_separator::deref_finder; @@ -102,6 +108,8 @@ struct RenameLocalVisitor<'tcx> { tcx: TyCtxt<'tcx>, } +const VARIANT_UNRESUMED: VariantIdx = VariantIdx::from_usize(CoroutineArgs::UNRESUMED); + impl<'tcx> MutVisitor<'tcx> for RenameLocalVisitor<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx @@ -159,7 +167,7 @@ impl<'tcx> MutVisitor<'tcx> for SelfArgVisitor<'tcx> { } } -#[tracing::instrument(level = "trace", skip(tcx))] +#[instrument(level = "trace", skip(tcx))] fn replace_base<'tcx>(place: &mut Place<'tcx>, new_base: Place<'tcx>, tcx: TyCtxt<'tcx>) { place.local = new_base.local; @@ -167,7 +175,7 @@ fn replace_base<'tcx>(place: &mut Place<'tcx>, new_base: Place<'tcx>, tcx: TyCtx new_projection.append(&mut place.projection.to_vec()); place.projection = tcx.mk_place_elems(&new_projection); - tracing::trace!(?place); + trace!(?place); } const SELF_ARG: Local = Local::from_u32(1); @@ -201,7 +209,7 @@ struct TransformVisitor<'tcx> { storage_liveness: IndexVec>>, // A list of suspension points, generated during the transform - suspension_points: Vec>, + suspension_points: IndexVec>, // The set of locals that have no `StorageLive`/`StorageDead` annotations. always_live_locals: DenseBitSet, @@ -272,7 +280,7 @@ impl<'tcx> TransformVisitor<'tcx> { // `core::ops::CoroutineState` only has single element tuple variants, // so we can just write to the downcasted first field and then set the // discriminant to the appropriate variant. - #[tracing::instrument(level = "trace", skip(self, statements))] + #[instrument(level = "trace", skip(self, statements))] fn make_state( &self, val: Operand<'tcx>, @@ -347,7 +355,7 @@ impl<'tcx> TransformVisitor<'tcx> { } // Create a Place referencing a coroutine struct field - #[tracing::instrument(level = "trace", skip(self), ret)] + #[instrument(level = "trace", skip(self), ret)] fn make_field(&self, variant_index: VariantIdx, idx: FieldIdx, ty: Ty<'tcx>) -> Place<'tcx> { let self_place = Place::from(SELF_ARG); let base = self.tcx.mk_place_downcast_unnamed(self_place, variant_index); @@ -358,7 +366,7 @@ impl<'tcx> TransformVisitor<'tcx> { } // Create a statement which changes the discriminant - #[tracing::instrument(level = "trace", skip(self))] + #[instrument(level = "trace", skip(self))] fn set_discr(&self, state_disc: VariantIdx, source_info: SourceInfo) -> Statement<'tcx> { let self_place = Place::from(SELF_ARG); Statement::new( @@ -371,7 +379,7 @@ impl<'tcx> TransformVisitor<'tcx> { } // Create a statement which reads the discriminant into a temporary - #[tracing::instrument(level = "trace", skip(self, body))] + #[instrument(level = "trace", skip(self, body))] fn get_discr(&self, body: &mut Body<'tcx>) -> (Statement<'tcx>, Place<'tcx>) { let temp_decl = LocalDecl::new(self.discr_ty, body.span); let local_decls_len = body.local_decls.push(temp_decl); @@ -386,7 +394,7 @@ impl<'tcx> TransformVisitor<'tcx> { } /// Swaps all references of `old_local` and `new_local`. - #[tracing::instrument(level = "trace", skip(self, body))] + #[instrument(level = "trace", skip(self, body))] fn replace_local(&mut self, old_local: Local, new_local: Local, body: &mut Body<'tcx>) { body.local_decls.swap(old_local, new_local); @@ -405,20 +413,30 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { self.tcx } - #[tracing::instrument(level = "trace", skip(self), ret)] + #[instrument(level = "trace", skip(self), ret)] fn visit_local(&mut self, local: &mut Local, _: PlaceContext, _location: Location) { assert!(!self.remap.contains(*local)); } - #[tracing::instrument(level = "trace", skip(self), ret)] + #[instrument(level = "trace", skip(self), ret)] fn visit_place(&mut self, place: &mut Place<'tcx>, _: PlaceContext, _location: Location) { // Replace an Local in the remap with a coroutine struct access if let Some(&Some((ty, variant_index, idx))) = self.remap.get(place.local) { replace_base(place, self.make_field(variant_index, idx, ty), self.tcx); + } else if let Place { local: ty::CAPTURE_STRUCT_LOCAL, projection } = *place + && let [first @ ProjectionElem::Field(..), rest @ ..] = &**projection + { + let projections: Vec<_> = [ProjectionElem::Downcast(None, VARIANT_UNRESUMED), *first] + .into_iter() + .chain(rest.iter().copied()) + .collect(); + let new_place = + Place::from(ty::CAPTURE_STRUCT_LOCAL).project_deeper(&projections, self.tcx); + *place = new_place; } } - #[tracing::instrument(level = "trace", skip(self, stmt), ret)] + #[instrument(level = "trace", skip(self, stmt), ret)] fn visit_statement(&mut self, stmt: &mut Statement<'tcx>, location: Location) { // Remove StorageLive and StorageDead statements for remapped locals if let StatementKind::StorageLive(l) | StatementKind::StorageDead(l) = stmt.kind @@ -429,7 +447,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { self.super_statement(stmt, location); } - #[tracing::instrument(level = "trace", skip(self, term), ret)] + #[instrument(level = "trace", skip(self, term), ret)] fn visit_terminator(&mut self, term: &mut Terminator<'tcx>, location: Location) { if let TerminatorKind::Return = term.kind { // `visit_basic_block_data` introduces `Return` terminators which read `RETURN_PLACE`. @@ -439,7 +457,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { self.super_terminator(term, location); } - #[tracing::instrument(level = "trace", skip(self, data), ret)] + #[instrument(level = "trace", skip(self, data), ret)] fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) { match data.terminator().kind { TerminatorKind::Return => { @@ -511,20 +529,20 @@ fn make_aggregate_adt<'tcx>( Rvalue::Aggregate(Box::new(AggregateKind::Adt(def_id, variant_idx, args, None, None)), operands) } -#[tracing::instrument(level = "trace", skip(tcx, body))] +#[instrument(level = "trace", skip(tcx, body))] fn make_coroutine_state_argument_indirect<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let coroutine_ty = body.local_decls[SELF_ARG].ty; + let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let ref_coroutine_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, coroutine_ty); // Replace the by value coroutine argument - body.local_decls[SELF_ARG].ty = ref_coroutine_ty; + body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty = ref_coroutine_ty; // Add a deref to accesses of the coroutine state SelfArgVisitor::new(tcx, tcx.mk_place_deref(SELF_ARG.into())).visit_body(body); } -#[tracing::instrument(level = "trace", skip(tcx, body))] +#[instrument(level = "trace", skip(tcx, body))] fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let coroutine_ty = body.local_decls[SELF_ARG].ty; @@ -536,7 +554,7 @@ fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body let pin_ref_coroutine_ty = Ty::new_adt(tcx, pin_adt_ref, args); // Replace the by ref coroutine argument - body.local_decls[SELF_ARG].ty = pin_ref_coroutine_ty; + body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty = pin_ref_coroutine_ty; let unpinned_local = body.local_decls.push(LocalDecl::new(ref_coroutine_ty, body.span)); @@ -586,7 +604,7 @@ fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body /// The async lowering step and the type / lifetime inference / checking are /// still using the `ResumeTy` indirection for the time being, and that indirection /// is removed here. After this transform, the coroutine body only knows about `&mut Context<'_>`. -#[tracing::instrument(level = "trace", skip(tcx, body), ret)] +#[instrument(level = "trace", skip(tcx, body), ret)] fn transform_async_context<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> Ty<'tcx> { let context_mut_ref = Ty::new_task_context(tcx); @@ -640,7 +658,7 @@ fn eliminate_get_context_call<'tcx>(bb_data: &mut BasicBlockData<'tcx>) -> Local } #[cfg_attr(not(debug_assertions), allow(unused))] -#[tracing::instrument(level = "trace", skip(tcx, body), ret)] +#[instrument(level = "trace", skip(tcx, body), ret)] fn replace_resume_ty_local<'tcx>( tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, @@ -677,15 +695,17 @@ fn transform_gen_context<'tcx>(body: &mut Body<'tcx>) { body.arg_count = 1; } +#[derive(Debug)] struct LivenessInfo { /// Which locals are live across any suspension point. saved_locals: CoroutineSavedLocals, /// The set of saved locals live at each suspension point. - live_locals_at_suspension_points: Vec>, + live_locals_at_suspension_points: + IndexVec>, /// Parallel vec to the above with SourceInfo for each yield terminator. - source_info_at_suspension_points: Vec, + source_info_at_suspension_points: IndexVec, /// For every saved local, the set of other saved locals that are /// storage-live at the same time as this local. We cannot overlap locals in @@ -697,6 +717,11 @@ struct LivenessInfo { storage_liveness: IndexVec>>, } +rustc_index::newtype_index! { + #[debug_format = "suspend_{}"] + struct SuspensionPointIdx {} +} + /// Computes which locals have to be stored in the state-machine for the /// given coroutine. /// @@ -705,16 +730,19 @@ struct LivenessInfo { /// case none exist, the local is considered to be always live. /// - a local has to be stored if it is either directly used after the /// the suspend point, or if it is live and has been previously borrowed. -#[tracing::instrument(level = "trace", skip(tcx, body))] +/// - finally, upvars conflict each other in storage; these `nr_upvars` +/// upvars are assigned to the last "phantom" saved locals. +#[instrument(level = "trace", skip(tcx, body))] fn locals_live_across_suspend_points<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - always_live_locals: &DenseBitSet, + always_live_locals: DenseBitSet, + nr_upvars: usize, movable: bool, ) -> LivenessInfo { // Calculate when MIR locals have live storage. This gives us an upper bound of their // lifetimes. - let mut storage_live = MaybeStorageLive::new(std::borrow::Cow::Borrowed(always_live_locals)) + let mut storage_live = MaybeStorageLive::new(std::borrow::Cow::Borrowed(&always_live_locals)) .iterate_to_fixpoint(tcx, body, None) .into_results_cursor(body); @@ -747,11 +775,12 @@ fn locals_live_across_suspend_points<'tcx>( MaybeLiveLocals.iterate_to_fixpoint(tcx, body, Some("coroutine")).into_results_cursor(body); let mut storage_liveness_map = IndexVec::from_elem(None, &body.basic_blocks); - let mut live_locals_at_suspension_points = Vec::new(); - let mut source_info_at_suspension_points = Vec::new(); + let mut live_locals_at_suspension_points = IndexVec::::default(); + let mut source_info_at_suspension_points = IndexVec::default(); let mut live_locals_at_any_suspension_point = DenseBitSet::new_empty(body.local_decls.len()); - for (block, data) in body.basic_blocks.iter_enumerated() { + for &block in body.basic_blocks.reverse_postorder() { + let data = &body.basic_blocks[block]; if let TerminatorKind::Yield { .. } = data.terminator().kind { let loc = Location { block, statement_index: data.statements.len() }; @@ -785,7 +814,7 @@ fn locals_live_across_suspend_points<'tcx>( live_locals.intersect(requires_storage_cursor.get()); // The coroutine argument is ignored. - live_locals.remove(SELF_ARG); + live_locals.remove(ty::CAPTURE_STRUCT_LOCAL); debug!("loc = {:?}, live_locals = {:?}", loc, live_locals); @@ -815,6 +844,35 @@ fn locals_live_across_suspend_points<'tcx>( &mut requires_storage.analysis, &requires_storage.results, ); + let storage_conflicts = if nr_upvars > 0 { + let nr_true_saved_locals = saved_locals.count(); + let nr_total_saved_locals = nr_true_saved_locals + nr_upvars; + let mut enlarged_storage_conflicts = + BitMatrix::new(nr_total_saved_locals, nr_total_saved_locals); + let mut upvars = DenseBitSet::new_empty(nr_total_saved_locals); + let mut ineligibles = upvars.clone(); + for (saved_local, local) in saved_locals.iter_enumerated() { + if always_live_locals.contains(local) { + ineligibles.insert(saved_local); + } + } + upvars.union(&ineligibles); + for row in storage_conflicts.rows() { + for column in storage_conflicts.iter(row) { + enlarged_storage_conflicts.insert(row, column); + } + } + for upvar in nr_true_saved_locals..nr_total_saved_locals { + let upvar = CoroutineSavedLocal::from_usize(upvar); + enlarged_storage_conflicts.union_row_with(&upvars, upvar); + } + for ineligible in ineligibles.iter() { + enlarged_storage_conflicts.union_row_with(&upvars, ineligible); + } + enlarged_storage_conflicts + } else { + storage_conflicts + }; LivenessInfo { saved_locals, @@ -830,6 +888,7 @@ fn locals_live_across_suspend_points<'tcx>( /// `CoroutineSavedLocal` is indexed in terms of the elements in this set; /// i.e. `CoroutineSavedLocal::new(1)` corresponds to the second local /// included in this set. +#[derive(Debug)] struct CoroutineSavedLocals(DenseBitSet); impl CoroutineSavedLocals { @@ -862,7 +921,7 @@ impl CoroutineSavedLocals { } } -impl ops::Deref for CoroutineSavedLocals { +impl Deref for CoroutineSavedLocals { type Target = DenseBitSet; fn deref(&self) -> &Self::Target { @@ -981,10 +1040,20 @@ impl StorageConflictVisitor<'_, '_> { } } -#[tracing::instrument(level = "trace", skip(liveness, body))] +#[derive(Debug, Copy, Clone)] +struct UpvarInfo { + name: Symbol, + span: Span, +} + +#[instrument(level = "debug", skip(liveness, body))] fn compute_layout<'tcx>( liveness: LivenessInfo, body: &Body<'tcx>, + upvar_tys: &[Ty<'tcx>], + upvar_infos: &[UpvarInfo], + pack: PackCoroutineLayout, + local_upvar_map: &IndexSlice, ) -> ( IndexVec, VariantIdx, FieldIdx)>>, CoroutineLayout<'tcx>, @@ -998,9 +1067,14 @@ fn compute_layout<'tcx>( storage_liveness, } = liveness; + // We need to later establish the map between upvars in UNRESUMED and locals in other states. + let local_upvar_map: UnordMap<_, _> = + local_upvar_map.iter_enumerated().map(|(field, &local)| (local, field)).collect(); + // Gather live local types and their indices. let mut locals = IndexVec::::new(); let mut tys = IndexVec::::new(); + let mut saved_local_upvar_map = UnordMap::default(); for (saved_local, local) in saved_locals.iter_enumerated() { debug!("coroutine saved local {:?} => {:?}", saved_local, local); @@ -1028,7 +1102,24 @@ fn compute_layout<'tcx>( debug!(?decl); tys.push(decl); + + if let Some(&field) = local_upvar_map.get(&local) { + saved_local_upvar_map.insert(field, saved_local); + } } + // These are the "saved locals" sourced from the UNRESUMED state. + let upvar_saved_locals: IndexVec = upvar_tys + .iter() + .zip(upvar_infos) + .map(|(&ty, info)| { + tys.push(CoroutineSavedTy { + ty, + source_info: SourceInfo::outermost(info.span), + ignore_for_traits: false, + }) + }) + .collect(); + debug!(?upvar_saved_locals); // Leave empty variants for the UNRESUMED, RETURNED, and POISONED states. // In debuginfo, these will correspond to the beginning (UNRESUMED) or end @@ -1045,12 +1136,14 @@ fn compute_layout<'tcx>( // Build the coroutine variant field list. // Create a map from local indices to coroutine struct indices. + let variant_fields: [_; CoroutineArgs::RESERVED_VARIANTS] = + [upvar_saved_locals.clone(), IndexVec::new(), IndexVec::new()]; let mut variant_fields: IndexVec> = - iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + variant_fields.into_iter().collect(); let mut remap = IndexVec::from_elem_n(None, saved_locals.domain_size()); - for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter().enumerate() { + for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter_enumerated() { let variant_index = - VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx); + VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx.as_usize()); let mut fields = IndexVec::new(); for (idx, saved_local) in live_locals.iter().enumerate() { fields.push(saved_local); @@ -1066,25 +1159,39 @@ fn compute_layout<'tcx>( } debug!("coroutine variant_fields = {:?}", variant_fields); debug!("coroutine storage_conflicts = {:#?}", storage_conflicts); + debug!(remap = ?remap.debug_map_view_compact()); + debug!(locals = ?locals.debug_map_view()); let mut field_names = IndexVec::from_elem(None, &tys); for var in &body.var_debug_info { - let VarDebugInfoContents::Place(place) = &var.value else { continue }; - let Some(local) = place.as_local() else { continue }; - let Some(&Some((_, variant, field))) = remap.get(local) else { - continue; - }; - - let saved_local = variant_fields[variant][field]; - field_names.get_or_insert_with(saved_local, || var.name); + debug!(?var); + if let VarDebugInfoContents::Place(place) = &var.value + && let Some(local) = place.local_or_deref_local() + && let Some(&Some((_, variant, field))) = remap.get(local) + { + let saved_local = variant_fields[variant][field]; + field_names.get_or_insert_with(saved_local, || var.name); + } + } + for (capture, &saved_local) in upvar_infos.iter().zip(&upvar_saved_locals) { + field_names.get_or_insert_with(saved_local, || capture.name); } + debug!(field_names = ?field_names.debug_map_view()); + let relocated_upvars = upvar_saved_locals + .iter_enumerated() + .filter_map(|(field, &source)| { + saved_local_upvar_map.get(&field).map(|&dest| (source, dest)) + }) + .collect(); let layout = CoroutineLayout { field_tys: tys, field_names, variant_fields, variant_source_info, storage_conflicts, + relocated_upvars, + pack, }; debug!(?remap); debug!(?layout); @@ -1260,7 +1367,7 @@ fn generate_poison_block_and_redirect_unwinds_there<'tcx>( } } -#[tracing::instrument(level = "trace", skip(tcx, transform, body))] +#[instrument(level = "trace", skip(tcx, transform, body))] fn create_coroutine_resume_function<'tcx>( tcx: TyCtxt<'tcx>, transform: TransformVisitor<'tcx>, @@ -1354,7 +1461,7 @@ impl Operation { } } -#[tracing::instrument(level = "trace", skip(transform, body))] +#[instrument(level = "trace", skip(transform, body))] fn create_cases<'tcx>( body: &mut Body<'tcx>, transform: &TransformVisitor<'tcx>, @@ -1410,14 +1517,20 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>( def_id: LocalDefId, ) -> Option> { let (body, _) = tcx.mir_promoted(def_id); - let body = body.borrow(); - let body = &*body; + let body = &*body.borrow(); // The first argument is the coroutine type passed by value let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; - let movable = match *coroutine_ty.kind() { - ty::Coroutine(def_id, _) => tcx.coroutine_movability(def_id) == hir::Movability::Movable, + let (movable, upvar_tys, upvar_infos) = match *coroutine_ty.kind() { + ty::Coroutine(def_id, args) => ( + matches!(tcx.coroutine_movability(def_id), hir::Movability::Movable), + args.as_coroutine().upvar_tys(), + tcx.closure_captures(def_id.expect_local()) + .iter() + .map(|info| UpvarInfo { name: info.var_ident.name, span: info.var_ident.span }) + .collect::>(), + ), ty::Error(_) => return None, _ => span_bug!(body.span, "unexpected coroutine type {}", coroutine_ty), }; @@ -1425,13 +1538,23 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>( // The witness simply contains all locals live across suspend points. let always_live_locals = always_storage_live_locals(body); - let liveness_info = locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); + debug!(?always_live_locals); + let liveness_info = + locals_live_across_suspend_points(tcx, body, always_live_locals, upvar_tys.len(), movable); // Extract locals which are live across suspension point into `layout` // `remap` gives a mapping from local indices onto coroutine struct indices // `storage_liveness` tells us which locals have live storage at suspension points - let (_, coroutine_layout, _) = compute_layout(liveness_info, body); - + let (_, coroutine_layout, _) = compute_layout( + liveness_info, + body, + upvar_tys, + &upvar_infos, + // Layout packing only happens at state transformation + // and we do not reflect this in analysis + PackCoroutineLayout::No, + IndexSlice::empty(), + ); check_suspend_tys(tcx, &coroutine_layout, body); check_field_tys_sized(tcx, &coroutine_layout, def_id); @@ -1486,25 +1609,58 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { // This only applies to coroutines return; }; - tracing::trace!(def_id = ?body.source.def_id()); + trace!(def_id = ?body.source.def_id()); let old_ret_ty = body.return_ty(); assert!(body.coroutine_drop().is_none() && body.coroutine_drop_async().is_none()); - - if let Some(dumper) = MirDumper::new(tcx, "coroutine_before", body) { - dumper.dump_mir(body); - } - // The first argument is the coroutine type passed by value - let coroutine_ty = body.local_decls.raw[1].ty; + let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let coroutine_kind = body.coroutine_kind().unwrap(); // Get the discriminant type and args which typeck computed - let ty::Coroutine(_, args) = coroutine_ty.kind() else { + let &ty::Coroutine(def_id, args) = coroutine_ty.kind() else { tcx.dcx().span_bug(body.span, format!("unexpected coroutine type {coroutine_ty}")); }; - let discr_ty = args.as_coroutine().discr_ty(tcx); + let coroutine_args = args.as_coroutine(); + let discr_ty = coroutine_args.discr_ty(tcx); + let upvar_tys = coroutine_args.upvar_tys(); + let local_upvar_map = if !matches!( + tcx.sess.opts.unstable_opts.pack_coroutine_layout, + rustc_session::config::PackCoroutineLayout::No + ) { + RelocateUpvars.run(tcx, def_id, upvar_tys, body) + } else { + Default::default() + }; + + if let Some(dumper) = MirDumper::new(tcx, "coroutine_before", body) { + dumper.dump_mir(body); + } + + let upvar_infos = match body.source.instance { + ty::InstanceKind::AsyncDropGlue(..) => { + smallvec![UpvarInfo { name: kw::SelfLower, span: DUMMY_SP }] + } + ty::InstanceKind::Item(_) => tcx + .closure_captures(def_id.expect_local()) + .iter() + .map(|info| UpvarInfo { name: info.var_ident.name, span: info.var_ident.span }) + .collect::>(), + ty::InstanceKind::Intrinsic(..) + | ty::InstanceKind::VTableShim(..) + | ty::InstanceKind::ReifyShim(..) + | ty::InstanceKind::FnPtrShim(..) + | ty::InstanceKind::Virtual(..) + | ty::InstanceKind::ClosureOnceShim { .. } + | ty::InstanceKind::ConstructCoroutineInClosureShim { .. } + | ty::InstanceKind::ThreadLocalShim(..) + | ty::InstanceKind::FutureDropPollShim(..) + | ty::InstanceKind::DropGlue(..) + | ty::InstanceKind::CloneShim(..) + | ty::InstanceKind::AsyncDropGlueCtorShim(..) + | ty::InstanceKind::FnPtrAddrShim(..) => unreachable!(), + }; let new_ret_ty = match coroutine_kind { CoroutineKind::Desugared(CoroutineDesugaring::Async, _) => { @@ -1559,9 +1715,14 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { } let always_live_locals = always_storage_live_locals(body); - let movable = coroutine_kind.movability() == hir::Movability::Movable; - let liveness_info = - locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); + let movable = matches!(coroutine_kind.movability(), hir::Movability::Movable); + let liveness_info = locals_live_across_suspend_points( + tcx, + body, + always_live_locals.clone(), + upvar_tys.len(), + movable, + ); if tcx.sess.opts.unstable_opts.validate_mir { let mut vis = EnsureCoroutineFieldAssignmentsNeverAlias { @@ -1576,14 +1737,21 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { // Extract locals which are live across suspension point into `layout` // `remap` gives a mapping from local indices onto coroutine struct indices // `storage_liveness` tells us which locals have live storage at suspension points - let (remap, layout, storage_liveness) = compute_layout(liveness_info, body); + let (remap, layout, storage_liveness) = compute_layout( + liveness_info, + body, + upvar_tys, + &upvar_infos, + tcx.sess.opts.unstable_opts.pack_coroutine_layout, + &local_upvar_map, + ); let can_return = can_return(tcx, body, body.typing_env(tcx)); // We rename RETURN_PLACE which has type mir.return_ty to new_ret_local // RETURN_PLACE then is a fresh unused local with type ret_ty. let new_ret_local = body.local_decls.push(LocalDecl::new(new_ret_ty, body.span)); - tracing::trace!(?new_ret_local); + trace!(?new_ret_local); // Run the transformation which converts Places from Local to coroutine struct // accesses for locals in `remap`. @@ -1596,7 +1764,7 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { remap, storage_liveness, always_live_locals, - suspension_points: Vec::new(), + suspension_points: IndexVec::default(), discr_ty, new_ret_local, old_ret_ty, diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs index 951ff69c19e3e..06e0cfc08e57c 100644 --- a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs +++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs @@ -69,7 +69,6 @@ use rustc_abi::{FieldIdx, VariantIdx}; use rustc_data_structures::steal::Steal; -use rustc_data_structures::unord::UnordMap; use rustc_hir as hir; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LocalDefId}; @@ -78,8 +77,18 @@ use rustc_middle::bug; use rustc_middle::hir::place::{Projection, ProjectionKind}; use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::{self, MirDumper}; +use rustc_middle::ty::data_structures::IndexMap; use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt, TypeVisitableExt}; +struct CaptureInfo<'tcx> { + /// Field index of the capture in the parent coroutine structure + remapped_idx: FieldIdx, + /// Type of the capture in the parent coroutine structure + remapped_ty: Ty<'tcx>, + peel_deref: bool, + bridging_projections: Vec>, +} + pub(crate) fn coroutine_by_move_body_def_id<'tcx>( tcx: TyCtxt<'tcx>, coroutine_def_id: LocalDefId, @@ -126,23 +135,27 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( .tuple_fields() .len(); - let field_remapping: UnordMap<_, _> = ty::analyze_coroutine_closure_captures( + let field_remapping: IndexMap<_, _> = ty::analyze_coroutine_closure_captures( tcx.closure_captures(parent_def_id).iter().copied(), tcx.closure_captures(coroutine_def_id).iter().skip(num_args).copied(), |(parent_field_idx, parent_capture), (child_field_idx, child_capture)| { // Store this set of additional projections (fields and derefs). // We need to re-apply them later. - let mut child_precise_captures = - child_capture.place.projections[parent_capture.place.projections.len()..].to_vec(); + let child_precise_captures = child_capture.place.projections + [parent_capture.place.projections.len()..] + .iter() + .copied(); // If the parent capture is by-ref, then we need to apply an additional // deref before applying any further projections to this place. - if parent_capture.is_by_ref() { - child_precise_captures.insert( - 0, - Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref }, - ); - } + let bridging_projections = if parent_capture.is_by_ref() { + [Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref }] + .into_iter() + .chain(child_precise_captures) + .collect() + } else { + child_precise_captures.collect() + }; // If the child capture is by-ref, then we need to apply a "ref" // projection (i.e. `&`) at the end. But wait! We don't have that // as a projection kind. So instead, we can apply its dual and @@ -168,8 +181,8 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( // Finally, store the type of the parent's captured place. We need // this when building the field projection in the MIR body later on. - let mut parent_capture_ty = parent_capture.place.ty(); - parent_capture_ty = match parent_capture.info.capture_kind { + let parent_capture_ty = parent_capture.place.ty(); + let remapped_ty = match parent_capture.info.capture_kind { ty::UpvarCapture::ByValue | ty::UpvarCapture::ByUse => parent_capture_ty, ty::UpvarCapture::ByRef(kind) => Ty::new_ref( tcx, @@ -181,19 +194,19 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( Some(( FieldIdx::from_usize(child_field_idx + num_args), - ( - FieldIdx::from_usize(parent_field_idx + num_args), - parent_capture_ty, + CaptureInfo { + remapped_idx: FieldIdx::from_usize(parent_field_idx + num_args), + remapped_ty, peel_deref, - child_precise_captures, - ), + bridging_projections, + }, )) }, ) .flatten() .collect(); - if coroutine_kind == ty::ClosureKind::FnOnce { + if matches!(coroutine_kind, ty::ClosureKind::FnOnce) { assert_eq!(field_remapping.len(), tcx.closure_captures(parent_def_id).len()); // The by-move body is just the body :) return coroutine_def_id.to_def_id(); @@ -212,6 +225,9 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( ); let mut by_move_body = body.clone(); + if let Some(mir_dumper) = MirDumper::new(tcx, "built", &body) { + mir_dumper.dump_mir(&body); + } MakeByMoveBody { tcx, field_remapping, by_move_coroutine_ty }.visit_body(&mut by_move_body); // This path is unique since we're in a query so we'll only be called once with `parent_def_id` @@ -254,7 +270,7 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( struct MakeByMoveBody<'tcx> { tcx: TyCtxt<'tcx>, - field_remapping: UnordMap, bool, Vec>)>, + field_remapping: IndexMap>, by_move_coroutine_ty: Ty<'tcx>, } @@ -275,8 +291,12 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { if place.local == ty::CAPTURE_STRUCT_LOCAL && let Some((&mir::ProjectionElem::Field(idx, _), projection)) = place.projection.split_first() - && let Some(&(remapped_idx, remapped_ty, peel_deref, ref bridging_projections)) = - self.field_remapping.get(&idx) + && let Some(&CaptureInfo { + remapped_idx, + remapped_ty, + peel_deref, + ref bridging_projections, + }) = self.field_remapping.get(&idx) { // As noted before, if the parent closure captures a field by value, and // the child captures a field by ref, then for the by-move body we're @@ -353,7 +373,7 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { local: ty::CAPTURE_STRUCT_LOCAL, projection: [mir::ProjectionElem::Field(idx, _)], } = place.as_ref() - && let Some(&(_, _, true, _)) = self.field_remapping.get(&idx) + && let Some(CaptureInfo { peel_deref: true, .. }) = self.field_remapping.get(idx) { statement.kind = mir::StatementKind::Nop; } diff --git a/compiler/rustc_mir_transform/src/coroutine/drop.rs b/compiler/rustc_mir_transform/src/coroutine/drop.rs index 2699a051a8fea..2a122e989f5ba 100644 --- a/compiler/rustc_mir_transform/src/coroutine/drop.rs +++ b/compiler/rustc_mir_transform/src/coroutine/drop.rs @@ -233,8 +233,14 @@ pub(super) fn has_expandable_async_drops<'tcx>( if body[bb].is_cleanup { continue; } - let TerminatorKind::Drop { place, target: _, unwind: _, replace: _, drop: _, async_fut } = - body[bb].terminator().kind + let TerminatorKind::Drop { + place, + target: _, + unwind: _, + replace: _, + drop: _, + async_fut: Some(_), + } = body[bb].terminator().kind else { continue; }; @@ -242,12 +248,9 @@ pub(super) fn has_expandable_async_drops<'tcx>( if place_ty == coroutine_ty { continue; } - if async_fut.is_none() { - continue; - } return true; } - return false; + false } /// Expand Drop terminator for async drops into mainline poll-switch and dropline poll-switch @@ -513,7 +516,8 @@ pub(super) fn elaborate_coroutine_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body elaborate_drop( &mut elaborator, *source_info, - Place::from(SELF_ARG), + Place::from(SELF_ARG) + .project_deeper(&[ProjectionElem::Downcast(None, VARIANT_UNRESUMED)], tcx), (), *target, unwind, diff --git a/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs new file mode 100644 index 0000000000000..e99b746cbd80f --- /dev/null +++ b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs @@ -0,0 +1,436 @@ +//! MIR rewrite pass to relocate upvars into native locals in the coroutine body +//! +//! # Summary +//! The current contract of coroutine upvars is as follows. +//! Coroutines are constructed, initially in state UNRESUMED, by copying or moving +//! captures into the `struct`-fields, which are also called prefix fields, +//! taking necessary references as per capture specification. +//! +//! ```text +//! Low address High address +//! ┌─────────┬─────────┬─────┬─────────────────────┬───────────────────────────────────────────────────────┬──────────────┐ +//! │ │ │ │ │ │ │ +//! │ Upvar 1 │ Upvar 2 │ ... │ Coroutine State Tag │ Ineligibles, aka. saved locals alive across 2+ states │ Other states │ +//! │ │ │ │ │ │ │ +//! └─────────┴─────────┴─────┴─────────────────────┴───────────────────────────────────────────────────────┴──────────────┘ +//! ``` +//! +//! In case some upvars are large and short-lived, the classic layout scheme can be wasteful. +//! One way to reduce the memory footprint is to +//! +//! This pass performs the following transformations. +//! 1. It generates a fresh batch of locals for each captured upvars. +//! +//! For each upvar, whether used or not, a fresh local is created with the same type. +//! The types respect the nature of the captures, being by-ref, by-ref-mut or by-value. +//! This is reflected in the results in the upvar analysis conducted in the HIR type-checking phase. +//! +//! 2. It replaces the places pointing into those upvars with places pointing into those locals instead +//! +//! Each place that starts with access into the coroutine structure `_1` is replaced with the fresh local as +//! the base. +//! Suppose we are to lower this coroutine into the MIR. +//! ```ignore (illustrative) +//! let mut captured = None; +//! let _ = #[coroutine] || { +//! yield (); +//! if let Some(inner) = &mut captured { +//! *inner = 42i32; // (*) +//! } +//! }; +//! ``` +//! `captured` is the only capture, whose mutable borrow is formally allotted to the first field `_1.0: &mut i32`. +//! The highlighted line `(*)` should be lowered, roughly, into MIR `(*_1.0) = const 42i32;`. +//! Now, by application of this pass, we create a new local `_4: &mut i32` and we perform the following +//! code transformation. +//! +//! A new block is constructed to just perform the relocation of this mutable borrow. +//! This block is inserted to the very beginning of the coroutine body control flow, +//! so that this is executed before any proper coroutine code as it transits from `UNRESUME` state to +//! any other state. +//! This "prologue" will look like the following. +//! ```ignore (illustrative) +//! StorageLive(_5); +//! StorageLive(_4); +//! _5 = move (_1.0); +//! _4 = move (_5); +//! StorageDead(_5); +//! ``` +//! Note that we also create a trampoline local `_5` of the same type. +//! +//! ### Intricacy around the trampoline local +//! The reason that we need the trampolines is because state transformation and coroutine +//! layout calculation is not aware of potential storage conflict between captures as struct fields +//! and other saved locals. +//! The only guarantee that we can work with is one where any coroutine layout calculator respects +//! the storage conflict contracts between *MIR locals*. +//! It is known that calculators do not consider struct fields, where captures reside, as MIR locals. +//! This is the source of potential memory overlaps. +//! For instance, in a hypothetical situation, +//! - `_1.0` is relocated to `_4`; +//! - `_1.1` is relocated to `_6`; +//! - `_4` and `_6` remains live at one of the first suspension state; +//! - `_4` occupies the same offset of `_1.1` and `_6` occupies the same offset of `_1.0` +//! as decided by some layout calculator; +//! In this scenario, without trampolining, the relocations introduce undefined behaviour. +//! +//! As a proposal for a future design, it is best that coroutine captures receive their own +//! MIR locals, possibly in a form of "function arguments" like `_1` itself. +//! The trampolining transformation already attempts to restore the semantics of MIR locals to +//! these captures and promoting them to "arguments" would make MIR safer to handle. +//! +//! One should note that this phase assumes that the initial built MIR respects the nature of captures. +//! For instance, if the upvar `_1.4` is instead a by-ref-mut capture of a value of type `T`, +//! this phase assumes that all access correctly built as operating on the place `(*_1.4)`. +//! Based on the assumption, this phase replaces `_1.4` with a fresh local `_34: &mut T` and +//! the correctness is still upheld. +//! +//! 3. It assembles an prologue to replace the current entry block. +//! +//! This prologue block transfers every captured upvar into its corresponding fresh local, *via scratch locals*. +//! The upvars are first completely moved into the scratch locals in batch, and then moved into the destination +//! locals in batch. +//! The reason is that it is possible that coroutine layout may change and the source memory location of +//! an upvar may not necessarily be mapped exactly to the same place as in the `UNRESUMED` state. +//! This is very possible, because the coroutine layout scheme at this moment remains opaque, +//! other than the contract that a saved local has a stable internal offset throughout its liveness span. +//! +//! While the current coroutine layout ensures that the same saved local has stable offsets throughout its lifetime, +//! technically the upvar in `UNRESUMED` state and their fresh locals are different saved locals. +//! This scratch locals re-establish safety so that the correct data permutation can take place, +//! when a future coroutine layout calculator sees the permutation fit. + +use itertools::Itertools; +use rustc_abi::FieldIdx; +use rustc_hir::def_id::DefId; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::{IndexSlice, IndexVec}; +use rustc_middle::bug; +use rustc_middle::mir::visit::{MutVisitor, PlaceContext}; +use rustc_middle::mir::*; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_span::{DUMMY_SP, Ident, Span, Symbol}; +use smallvec::{SmallVec, smallvec}; +use tracing::{debug, instrument}; + +use crate::elaborate_drop::{Unwind, elaborate_drop}; +use crate::pass_manager::MirPass; +use crate::patch::MirPatch; +use crate::shim::DropShimElaborator; + +pub(crate) struct RelocateUpvars; + +struct UpvarSubstitution<'tcx> { + /// Newly minted local into which the upvar is moved + local: Local, + /// The temporary local that the prologue will permute the upvars with + reloc: Local, + /// Place into the capture structure where this upvar is found + upvar_place: Place<'tcx>, + /// The span of the captured upvar from the parent body + span: Span, + /// Name of the upvar + name: Symbol, +} + +struct SubstituteUpvarVisitor<'tcx, 'a> { + tcx: TyCtxt<'tcx>, + mappings: &'a IndexSlice>, +} + +impl<'tcx, 'a> MutVisitor<'tcx> for SubstituteUpvarVisitor<'tcx, 'a> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_place(&mut self, place: &mut Place<'tcx>, _context: PlaceContext, location: Location) { + if let Place { local: ty::CAPTURE_STRUCT_LOCAL, projection } = place + && let [ProjectionElem::Field(field_idx, _ty), rest @ ..] = &***projection + { + let Some(&UpvarSubstitution { local, .. }) = self.mappings.get(*field_idx) else { + bug!( + "SubstituteUpvar: found {field_idx:?} @ {location:?} but there is no upvar for it" + ) + }; + let new_place = Place::from(local).project_deeper(rest, self.tcx); + *place = new_place; + } + } + + fn visit_local(&mut self, local: &mut Local, context: PlaceContext, location: Location) { + if *local == ty::CAPTURE_STRUCT_LOCAL { + bug!("found a stray _1 at {location:?} in context {context:?}") + } + } +} + +#[derive(Debug, Clone, Copy)] +struct RelocationInfo { + ident: Option, + immutable: bool, + by_ref: bool, +} + +impl RelocateUpvars { + #[instrument(level = "debug", skip_all, fields(def_id = ?body.source))] + pub(crate) fn run<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + def_id: DefId, + upvar_tys: &ty::List>, + body: &mut Body<'tcx>, + ) -> IndexVec { + if upvar_tys.is_empty() { + debug!("no upvar, skipping"); + return Default::default(); + } + + let upvar_infos = match body.source.instance { + ty::InstanceKind::AsyncDropGlue(..) => { + smallvec![RelocationInfo { ident: None, immutable: true, by_ref: false }] + } + ty::InstanceKind::Item(_) => tcx + .closure_captures(def_id.expect_local()) + .iter() + .map(|info| RelocationInfo { + ident: Some(Ident::new(info.to_symbol(), info.var_ident.span)), + immutable: matches!(info.mutability, ty::Mutability::Not), + by_ref: matches!(info.info.capture_kind, ty::UpvarCapture::ByRef(..)), + }) + .collect::>(), + ty::InstanceKind::Intrinsic(..) + | ty::InstanceKind::VTableShim(..) + | ty::InstanceKind::ReifyShim(..) + | ty::InstanceKind::FnPtrShim(..) + | ty::InstanceKind::Virtual(..) + | ty::InstanceKind::ClosureOnceShim { .. } + | ty::InstanceKind::ConstructCoroutineInClosureShim { .. } + | ty::InstanceKind::ThreadLocalShim(..) + | ty::InstanceKind::FutureDropPollShim(..) + | ty::InstanceKind::DropGlue(..) + | ty::InstanceKind::CloneShim(..) + | ty::InstanceKind::AsyncDropGlueCtorShim(..) + | ty::InstanceKind::FnPtrAddrShim(..) => unreachable!(), + }; + + // HACK: in case `AddRetag` is already run, we have one `Retag` at the body entrance + // so we need to make sure that first `Retag` is run. + let retags_in_start_block = body.basic_blocks[START_BLOCK] + .statements + .iter() + .find_position(|stmt| !matches!(stmt.kind, StatementKind::Retag(_, _))) + .map(|(loc, _)| loc); + + if let Some(mir_dumper) = MirDumper::new(tcx, "RelocateUpvars", body) { + mir_dumper.set_disambiguator(&"before").dump_mir(body); + } + + let mut substitution_mapping = IndexVec::new(); + for (field_idx, (upvar_ty, &captured)) in upvar_tys.iter().zip(&upvar_infos).enumerate() { + let span = captured.ident.map_or(DUMMY_SP, |ident| ident.span); + let name = if let Some(ident) = captured.ident { + ident.name + } else { + Symbol::intern(&format!("_{}", field_idx)) + }; + + let immutable = if captured.immutable { Mutability::Not } else { Mutability::Mut }; + let mut local_decl = LocalDecl::new(upvar_ty, span); + **local_decl.local_info.as_mut().unwrap_crate_local() = + LocalInfo::User(BindingForm::Var(VarBindingForm { + binding_mode: rustc_ast::BindingMode( + if captured.by_ref { + rustc_ast::ByRef::Yes(immutable) + } else { + rustc_ast::ByRef::No + }, + immutable, + ), + opt_ty_info: None, + opt_match_place: None, + pat_span: span, + introductions: vec![], + })); + let local = body.local_decls.push(local_decl); + + local_decl = LocalDecl::new(upvar_ty, span); + **local_decl.local_info.as_mut().unwrap_crate_local() = LocalInfo::Boring; + let reloc = body.local_decls.push(local_decl); + + let field_idx = FieldIdx::from_usize(field_idx); + let upvar_place = + tcx.mk_place_field(Place::from(ty::CAPTURE_STRUCT_LOCAL), field_idx, upvar_ty); + + substitution_mapping.push(UpvarSubstitution { local, reloc, upvar_place, span, name }); + } + rewrite_drop_coroutine_struct(tcx, body); + let local_upvar_map = substitution_mapping.iter().map(|sub| sub.local).collect(); + SubstituteUpvarVisitor { tcx, mappings: &substitution_mapping }.visit_body(body); + + insert_substitution_prologue(body, retags_in_start_block, &substitution_mapping); + patch_missing_storage_deads(body, &substitution_mapping); + hydrate_var_debug_info(body, &substitution_mapping); + if let Some(mir_dumper) = MirDumper::new(tcx, "RelocateUpvars", body) { + mir_dumper.set_disambiguator(&"after").dump_mir(body); + } + local_upvar_map + } +} + +impl<'tcx> MirPass<'tcx> for RelocateUpvars { + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let coroutine_ty = if body.yield_ty().is_some() + && let Some(decl) = body.local_decls.get(ty::CAPTURE_STRUCT_LOCAL) + { + decl.ty + } else { + // It fails the litmus test as a coroutine + bug!("RelocateUpvars can only be run on coroutines"); + }; + let (def_id, upvar_tys) = if let ty::Coroutine(def_id, args) = *coroutine_ty.kind() { + let args = args.as_coroutine(); + (def_id, args.upvar_tys()) + } else { + bug!("RelocateUpvars can only be run on coroutines"); + }; + self.run(tcx, def_id, upvar_tys, body); + } + + fn is_required(&self) -> bool { + true + } +} + +fn rewrite_drop_coroutine_struct<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + // Note that `elaborate_drops` only drops the upvars of a coroutine, and + // this is ok because `open_drop` can only be reached within that own + // coroutine's resume function. + let typing_env = body.typing_env(tcx); + + let mut elaborator = DropShimElaborator { + body, + patch: MirPatch::new(body), + tcx, + typing_env, + produce_async_drops: false, + }; + + for (block, block_data) in body.basic_blocks.iter_enumerated() { + let (target, unwind, source_info, dropline) = match block_data.terminator() { + Terminator { + source_info, + kind: TerminatorKind::Drop { place, target, unwind, replace: _, drop, async_fut: _ }, + } => { + if matches!(place.as_local(), Some(ty::CAPTURE_STRUCT_LOCAL)) { + (target, unwind, source_info, *drop) + } else { + continue; + } + } + _ => continue, + }; + let unwind = if block_data.is_cleanup { + Unwind::InCleanup + } else { + Unwind::To(match *unwind { + UnwindAction::Cleanup(tgt) => tgt, + UnwindAction::Continue => elaborator.patch.resume_block(), + UnwindAction::Unreachable => elaborator.patch.unreachable_cleanup_block(), + UnwindAction::Terminate(reason) => elaborator.patch.terminate_block(reason), + }) + }; + elaborate_drop( + &mut elaborator, + *source_info, + Place::from(ty::CAPTURE_STRUCT_LOCAL), + (), + *target, + unwind, + block, + dropline, + ); + } + elaborator.patch.apply(body); +} + +fn insert_substitution_prologue<'tcx>( + body: &mut Body<'tcx>, + retags_in_start_block: Option, + substitution_mapping: &IndexSlice>, +) { + // NOTE: START_BLOCK cannot have incoming edges + let mut stmts = Vec::with_capacity(5 * substitution_mapping.len()); + for &UpvarSubstitution { local, reloc, upvar_place, span, name: _ } in substitution_mapping { + // For each upvar-local _$i + let source_info = SourceInfo::outermost(span); + // StorageLive(_$i) + stmts.push(Statement::new(source_info, StatementKind::StorageLive(local))); + // Use a fresh local _$i' here, so as to avoid potential field permutation + // StorageLive(_$i') + stmts.push(Statement::new(source_info, StatementKind::StorageLive(reloc))); + // _$i' = move $ + stmts.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + reloc.into(), + Rvalue::Use(Operand::Move(upvar_place)), + ))), + )); + } + for &UpvarSubstitution { local, reloc, upvar_place: _, span, name: _ } in substitution_mapping { + let source_info = SourceInfo::outermost(span); + // _$i = move $i' + stmts.push(Statement::new( + source_info, + StatementKind::Assign(Box::new(( + local.into(), + Rvalue::Use(Operand::Move(reloc.into())), + ))), + )); + stmts.push(Statement::new(source_info, StatementKind::StorageDead(reloc))); + } + + let start_stmts = &mut body.basic_blocks.as_mut_preserves_cfg()[START_BLOCK].statements; + let tail_stmts: Vec<_> = + start_stmts.splice(retags_in_start_block.unwrap_or(0).., stmts).collect(); + start_stmts.extend(tail_stmts); +} + +/// We need to mark relocated upvars daed for correctness, +/// as previously just the entire capture structure was marked dead and +/// now we need to mark them dead individually. +fn patch_missing_storage_deads<'tcx>( + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + let mut upvar_locals = DenseBitSet::new_empty(body.local_decls.len()); + for subst in substitution_mapping { + upvar_locals.insert(subst.local); + } + let basic_blocks = body.basic_blocks.as_mut_preserves_cfg(); + for data in basic_blocks.iter_mut() { + if !data.is_cleanup && matches!(data.terminator().kind, TerminatorKind::Return) { + for local in upvar_locals.iter() { + data.statements.push(Statement::new( + data.terminator().source_info, + StatementKind::StorageDead(local), + )); + } + } + } +} + +fn hydrate_var_debug_info<'tcx>( + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + for subst in substitution_mapping { + body.var_debug_info.push(VarDebugInfo { + name: subst.name, + source_info: SourceInfo::outermost(subst.span), + composite: None, + value: VarDebugInfoContents::Place(Place::from(subst.local)), + argument_index: None, + }); + } +} diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 4625b20fd8900..b2f002f773e3b 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -128,7 +128,7 @@ declare_passes! { pub mod cleanup_post_borrowck : CleanupPostBorrowck; mod copy_prop : CopyProp; - mod coroutine : StateTransform; + mod coroutine : RelocateUpvars, StateTransform; mod coverage : InstrumentCoverage; mod ctfe_limit : CtfeLimit; mod dataflow_const_prop : DataflowConstProp; diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index ab09cdf787ee7..987510f714b53 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -311,12 +311,21 @@ fn run_passes_inner<'tcx>( if let Some(dumper) = dumper { dumper.set_disambiguator(&"after").dump_mir(body); } + let (dialect, phase_num) = body.phase.index(); if validate { - validate_body(tcx, body, format!("after pass {pass_name}")); + validate_body( + tcx, + body, + format!("after pass {pass_name} {dialect}-{phase_num}-{:03}", body.pass_count), + ); } if lint { - lint_body(tcx, body, format!("after pass {pass_name}")); + lint_body( + tcx, + body, + format!("after pass {pass_name} {dialect}-{phase_num}-{:03}", body.pass_count), + ); } body.pass_count += 1; diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 85e340c0a02ab..14075d362ef2d 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -49,7 +49,7 @@ impl<'tcx> MutVisitor<'tcx> for FixProxyFutureDropVisitor<'tcx> { _context: PlaceContext, _location: Location, ) { - if place.local == Local::from_u32(1) { + if place.local == ty::CAPTURE_STRUCT_LOCAL { if place.projection.len() == 1 { assert!(matches!( place.projection.first(), @@ -66,9 +66,8 @@ impl<'tcx> MutVisitor<'tcx> for FixProxyFutureDropVisitor<'tcx> { } } +#[instrument(level = "debug", skip(tcx))] fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<'tcx> { - debug!("make_shim({:?})", instance); - let mut result = match instance { ty::InstanceKind::Item(..) => bug!("item {:?} passed to make_shim", instance), ty::InstanceKind::VTableShim(def_id) => { @@ -195,6 +194,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body< tcx, &mut body, &[ + &crate::coroutine::RelocateUpvars, // We must always relocate to ensure valid MIR &mentioned_items::MentionedItems, &abort_unwinding_calls::AbortUnwindingCalls, &add_call_guards::CriticalCallEdges, diff --git a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs index a0f1260cd986d..a3b718b164086 100644 --- a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs +++ b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs @@ -11,6 +11,7 @@ use rustc_middle::ty::{self, EarlyBinder, Ty, TyCtxt, TypeVisitableExt}; use super::*; use crate::patch::MirPatch; +#[instrument(level = "debug", skip(tcx))] pub(super) fn build_async_destructor_ctor_shim<'tcx>( tcx: TyCtxt<'tcx>, def_id: DefId, @@ -39,12 +40,12 @@ pub(super) fn build_async_destructor_ctor_shim<'tcx>( } // build_drop_shim analog for async drop glue (for generated coroutine poll function) +#[instrument(level = "debug", skip(tcx))] pub(super) fn build_async_drop_shim<'tcx>( tcx: TyCtxt<'tcx>, def_id: DefId, ty: Ty<'tcx>, ) -> Body<'tcx> { - debug!("build_async_drop_shim(def_id={:?}, ty={:?})", def_id, ty); let ty::Coroutine(_, parent_args) = ty.kind() else { bug!(); }; @@ -193,6 +194,7 @@ pub(super) fn build_future_drop_poll_shim<'tcx>( // `async_drop_in_place::{closure}.poll()` is converted into `T.future_drop_poll()`. // Every coroutine has its `poll` (calculate yourself a little further) // and its `future_drop_poll` (drop yourself a little further). +#[instrument(level = "debug", skip_all, fields(span))] fn build_adrop_for_coroutine_shim<'tcx>( tcx: TyCtxt<'tcx>, proxy_ty: Ty<'tcx>, @@ -200,33 +202,43 @@ fn build_adrop_for_coroutine_shim<'tcx>( span: Span, instance: ty::InstanceKind<'tcx>, ) -> Body<'tcx> { - let ty::Coroutine(coroutine_def_id, impl_args) = impl_ty.kind() else { + debug!("proxy_ty={proxy_ty:#?}"); + debug!("impl_ty={impl_ty:#?}"); + debug!("instance={instance:#?}"); + let &ty::Coroutine(coroutine_def_id, impl_args) = impl_ty.kind() else { bug!("build_adrop_for_coroutine_shim not for coroutine impl type: ({:?})", instance); }; let proxy_ref = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, proxy_ty); - // taking _1.0 (impl from Pin) - let pin_proxy_layout_local = Local::new(1); - let source_info = SourceInfo::outermost(span); - // converting `(_1: Pin<&mut CorLayout>, _2: &mut Context<'_>) -> Poll<()>` - // into `(_1: Pin<&mut ProxyLayout>, _2: &mut Context<'_>) -> Poll<()>` - // let mut _x: &mut CorLayout = &*_1.0.0; - // Replace old _1.0 accesses into _x accesses; - let body = tcx.optimized_mir(*coroutine_def_id).future_drop_poll().unwrap(); + let body = tcx.optimized_mir(coroutine_def_id).future_drop_poll().unwrap(); let mut body: Body<'tcx> = EarlyBinder::bind(body.clone()).instantiate(tcx, impl_args); + if let Some(mir_dumper) = MirDumper::new(tcx, "build_adrop_for_coroutine_shim_before", &body) { + mir_dumper.set_disambiguator(&0).dump_mir(&body); + } + debug!(?body.source.instance, "before"); body.source.instance = instance; body.phase = MirPhase::Runtime(RuntimePhase::Initial); body.var_debug_info.clear(); + // Here we convert `(_1: Pin<&mut InnerCoroutine>, _2: &mut Context<'_>) -> Poll<()>` + // into `(_1: Pin<&mut ProxyCoroutine>, _2: &mut Context<'_>) -> Poll<()>`. + // **The Schematic** + // We make a new local `_x`: + // let mut _x: &mut InnerCoroutine = &* ((*_1.0) as variant#0).0; + // Then replace projections on `_1.0` into projections on `_x`. + let pin_proxy_layout_local = Local::new(1); + let source_info = SourceInfo::outermost(span); let pin_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, span)); let args = tcx.mk_args(&[proxy_ref.into()]); let pin_proxy_ref = Ty::new_adt(tcx, pin_adt_ref, args); + // This is the type of the vanilla `InnerCoroutine` let cor_ref = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, impl_ty); let proxy_ref_local = body.local_decls.push(LocalDecl::new(proxy_ref, span)); let cor_ref_local = body.local_decls.push(LocalDecl::new(cor_ref, span)); + // Batch replacement of `_1.0` with `_x` FixProxyFutureDropVisitor { tcx, replace_to: cor_ref_local }.visit_body(&mut body); - // Now changing first arg from Pin<&mut ImplCoroutine> to Pin<&mut ProxyCoroutine> + // Now changing first arg from Pin<&mut InnerCoroutine> to Pin<&mut ProxyCoroutine> body.local_decls[pin_proxy_layout_local] = LocalDecl::new(pin_proxy_ref, span); { @@ -250,11 +262,15 @@ fn build_adrop_for_coroutine_shim<'tcx>( if ty != proxy_ty { let ty_ptr = Ty::new_mut_ptr(tcx, ty); let impl_ptr_place = Place::from(cor_ptr_local).project_deeper( - &[PlaceElem::Deref, PlaceElem::Field(FieldIdx::ZERO, ty_ptr)], + &[ + PlaceElem::Deref, + PlaceElem::Downcast(None, VariantIdx::ZERO), + PlaceElem::Field(FieldIdx::ZERO, ty_ptr), + ], tcx, ); cor_ptr_local = body.local_decls.push(LocalDecl::new(ty_ptr, span)); - // _cor_ptr = _proxy.0.0 (... .0) + // _cor_ptr = ((*_proxy.0) as variant#0).0 (... .0) body.basic_blocks_mut()[START_BLOCK].statements.insert( idx, Statement::new( @@ -283,6 +299,9 @@ fn build_adrop_for_coroutine_shim<'tcx>( ), ); } + if let Some(mir_dumper) = MirDumper::new(tcx, "build_adrop_for_coroutine_shim", &body) { + mir_dumper.set_disambiguator(&0).dump_mir(&body); + } body } @@ -342,10 +361,16 @@ fn build_adrop_for_adrop_shim<'tcx>( proxy_ty.find_async_drop_impl_coroutine(tcx, |ty| { if ty != proxy_ty { let ty_ptr = Ty::new_mut_ptr(tcx, ty); - let impl_ptr_place = Place::from(cor_ptr_local) - .project_deeper(&[PlaceElem::Deref, PlaceElem::Field(FieldIdx::ZERO, ty_ptr)], tcx); + let impl_ptr_place = Place::from(cor_ptr_local).project_deeper( + &[ + PlaceElem::Deref, + PlaceElem::Downcast(None, VariantIdx::ZERO), + PlaceElem::Field(FieldIdx::ZERO, ty_ptr), + ], + tcx, + ); cor_ptr_local = locals.push(LocalDecl::new(ty_ptr, span)); - // _cor_ptr = _proxy.0.0 (... .0) + // _cor_ptr = ((*_proxy.0) as variant#0).0 (... .0) statements.push(Statement::new( source_info, StatementKind::Assign(Box::new(( @@ -421,5 +446,8 @@ fn build_adrop_for_adrop_shim<'tcx>( let source = MirSource::from_instance(instance); let mut body = new_body(source, blocks, locals, sig.inputs().len(), span); body.phase = MirPhase::Runtime(RuntimePhase::Initial); - return body; + if let Some(mir_dumper) = MirDumper::new(tcx, "build_adrop_for_adrop_shim", &body) { + mir_dumper.set_disambiguator(&0).dump_mir(&body); + } + body } diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index da31600e8324c..f34ad2e5e0b2b 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -451,7 +451,7 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyLocals { // Only bother running the `LocalUpdater` if we actually found locals to remove. if map.iter().any(Option::is_none) { // Update references to all vars and tmps now - let mut updater = LocalUpdater { map, tcx }; + let mut updater = LocalUpdater { map: &map, tcx }; updater.visit_body_preserves_cfg(body); body.local_decls.shrink_to_fit(); @@ -647,12 +647,12 @@ fn remove_unused_definitions_helper(used_locals: &mut UsedLocals, body: &mut Bod } } -struct LocalUpdater<'tcx> { - map: IndexVec>, +struct LocalUpdater<'tcx, 'a> { + map: &'a IndexSlice>, tcx: TyCtxt<'tcx>, } -impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx> { +impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx, '_> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 8ff6d567422b9..b3fe99cd76f46 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -3252,9 +3252,9 @@ pub(crate) mod dep_tracking { CrateType, DebugInfo, DebugInfoCompression, ErrorOutputType, FmtDebug, FunctionReturn, InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail, LtoCli, MirStripDebugInfo, NextSolverConfig, Offload, OomStrategy, OptLevel, OutFileName, - OutputType, OutputTypes, PatchableFunctionEntry, Polonius, RemapPathScopeComponents, - ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, - SymbolManglingVersion, WasiExecModel, + OutputType, OutputTypes, PackCoroutineLayout, PatchableFunctionEntry, Polonius, + RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, + SwitchWithOptPath, SymbolManglingVersion, WasiExecModel, }; use crate::lint; use crate::utils::NativeLib; @@ -3357,6 +3357,7 @@ pub(crate) mod dep_tracking { Polonius, InliningThreshold, FunctionReturn, + PackCoroutineLayout, Align, ); @@ -3611,6 +3612,20 @@ pub enum FunctionReturn { ThunkExtern, } +/// Layout optimisation for Coroutines +#[derive(Clone, Copy, PartialEq, Eq, Hash, HashStable_Generic, Debug, Default)] +#[derive(Decodable, Encodable)] +pub enum PackCoroutineLayout { + /// Keep coroutine captured variables throughout all states + #[default] + No, + + /// Allow coroutine captured variables that are used only once + /// before the first suspension to be freed up for storage + /// in all other suspension states + CapturesOnly, +} + /// Whether extra span comments are included when dumping MIR, via the `-Z mir-include-spans` flag. /// By default, only enabled in the NLL MIR dumps, and disabled in all other passes. #[derive(Clone, Copy, Default, PartialEq, Debug)] diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 6dd90546de1b0..63c2604f341ee 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -805,6 +805,7 @@ mod desc { pub(crate) const parse_panic_strategy: &str = "either `unwind`, `abort`, or `immediate-abort`"; pub(crate) const parse_on_broken_pipe: &str = "either `kill`, `error`, or `inherit`"; pub(crate) const parse_patchable_function_entry: &str = "either two comma separated integers (total_nops,prefix_nops), with prefix_nops <= total_nops, or one integer (total_nops)"; + pub(crate) const parse_pack_coroutine_layout: &str = "either `no` or `captures-only`"; pub(crate) const parse_opt_panic_strategy: &str = parse_panic_strategy; pub(crate) const parse_oom_strategy: &str = "either `panic` or `abort`"; pub(crate) const parse_relro_level: &str = "one of: `full`, `partial`, or `off`"; @@ -1951,6 +1952,18 @@ pub mod parse { true } + pub(crate) fn parse_pack_coroutine_layout( + slot: &mut PackCoroutineLayout, + v: Option<&str>, + ) -> bool { + *slot = match v { + Some("no") => PackCoroutineLayout::No, + Some("captures-only") => PackCoroutineLayout::CapturesOnly, + _ => return false, + }; + true + } + pub(crate) fn parse_inlining_threshold(slot: &mut InliningThreshold, v: Option<&str>) -> bool { match v { Some("always" | "yes") => { @@ -2504,6 +2517,8 @@ options! { "panic strategy for out-of-memory handling"), osx_rpath_install_name: bool = (false, parse_bool, [TRACKED], "pass `-install_name @rpath/...` to the macOS linker (default: no)"), + pack_coroutine_layout: PackCoroutineLayout = (PackCoroutineLayout::default(), parse_pack_coroutine_layout, [TRACKED], + "set strategy to pack coroutine state layout (default: no)"), packed_bundled_libs: bool = (false, parse_bool, [TRACKED], "change rlib format to store native libraries as archives"), panic_abort_tests: bool = (false, parse_bool, [TRACKED], diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs index 136a61598d50e..66fd74ce349e4 100644 --- a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs +++ b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs @@ -24,6 +24,7 @@ use rustc_hir::{ }; use rustc_infer::infer::{BoundRegionConversionTime, DefineOpaqueTypes, InferCtxt, InferOk}; use rustc_middle::middle::privacy::Level; +use rustc_middle::mir::CoroutineLayout; use rustc_middle::traits::IsConstable; use rustc_middle::ty::error::TypeError; use rustc_middle::ty::print::{ @@ -2492,18 +2493,26 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> { && let Some(coroutine_info) = self.tcx.mir_coroutine_witnesses(coroutine_did) { debug!(?coroutine_info); - 'find_source: for (variant, source_info) in - coroutine_info.variant_fields.iter().zip(&coroutine_info.variant_source_info) + 'find_source: for ((variant_idx, variant), source_info) in coroutine_info + .variant_fields + .iter_enumerated() + .zip(&coroutine_info.variant_source_info) + .rev() { debug!(?variant); for &local in variant { let decl = &coroutine_info.field_tys[local]; debug!(?decl); if ty_matches(ty::Binder::dummy(decl.ty)) && !decl.ignore_for_traits { - interior_or_upvar_span = Some(CoroutineInteriorOrUpvar::Interior( - decl.source_info.span, - Some((source_info.span, from_awaited_ty)), - )); + let span = decl.source_info.span; + if variant_idx == CoroutineLayout::UNRESUMED { + interior_or_upvar_span = Some(CoroutineInteriorOrUpvar::Upvar(span)); + } else { + interior_or_upvar_span = Some(CoroutineInteriorOrUpvar::Interior( + span, + Some((source_info.span, from_awaited_ty)), + )); + } break 'find_source; } } diff --git a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs index 2e60805cd10a5..8292f312cbcc2 100644 --- a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs +++ b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs @@ -1,6 +1,8 @@ +use rustc_abi::VariantIdx; use rustc_data_structures::fx::FxHashSet; use rustc_infer::traits::query::type_op::DropckOutlives; use rustc_middle::traits::query::{DropckConstraint, DropckOutlivesResult}; +use rustc_middle::ty::data_structures::IndexSet; use rustc_middle::ty::{self, EarlyBinder, ParamEnvAnd, Ty, TyCtxt}; use rustc_span::Span; use tracing::{debug, instrument}; @@ -354,7 +356,14 @@ pub fn dtorck_constraint_for_ty_inner<'tcx>( // FIXME(@lcnr): Why do we erase regions in the env here? Seems odd let typing_env = tcx.erase_and_anonymize_regions(typing_env); let needs_drop = tcx.mir_coroutine_witnesses(def_id).is_some_and(|witness| { - witness.field_tys.iter().any(|field| field.ty.needs_drop(tcx, typing_env)) + let upvar_locals: IndexSet<_> = + witness.variant_fields[VariantIdx::ZERO].iter().copied().collect(); + // As a reminder, upvars also appear in the UNRESUMED state-variant + witness + .field_tys + .iter_enumerated() + .filter_map(|(corsl, ty)| (!upvar_locals.contains(&corsl)).then_some(ty)) + .any(|field| field.ty.needs_drop(tcx, typing_env)) }); if needs_drop { // Pushing types directly to `constraints.outlives` is equivalent diff --git a/compiler/rustc_traits/src/coroutine_witnesses.rs b/compiler/rustc_traits/src/coroutine_witnesses.rs index 2544cd8a13cd8..5505d1646ee06 100644 --- a/compiler/rustc_traits/src/coroutine_witnesses.rs +++ b/compiler/rustc_traits/src/coroutine_witnesses.rs @@ -2,24 +2,40 @@ use rustc_infer::infer::TyCtxtInferExt; use rustc_infer::infer::canonical::query_response::make_query_region_constraints; use rustc_infer::infer::resolve::OpportunisticRegionResolver; use rustc_infer::traits::{Obligation, ObligationCause}; +use rustc_middle::mir::CoroutineLayout; +use rustc_middle::ty::data_structures::IndexSet; use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeVisitableExt, fold_regions}; use rustc_span::def_id::DefId; use rustc_trait_selection::traits::{ObligationCtxt, with_replaced_escaping_bound_vars}; +use tracing::instrument; /// Return the set of types that should be taken into account when checking /// trait bounds on a coroutine's internal state. This properly replaces /// `ReErased` with new existential bound lifetimes. +#[instrument(level = "debug", skip(tcx), ret)] pub(crate) fn coroutine_hidden_types<'tcx>( tcx: TyCtxt<'tcx>, def_id: DefId, ) -> ty::EarlyBinder<'tcx, ty::Binder<'tcx, ty::CoroutineWitnessTypes>>> { - let coroutine_layout = tcx.mir_coroutine_witnesses(def_id); + let Some(coroutine_layout) = tcx.mir_coroutine_witnesses(def_id) else { + return ty::EarlyBinder::bind(ty::Binder::dummy(ty::CoroutineWitnessTypes { + types: ty::List::empty(), + assumptions: ty::List::empty(), + })); + }; + let mut internal_fields: IndexSet<_> = + coroutine_layout.variant_fields.iter().skip(1).flat_map(|f| f.iter().copied()).collect(); + for upvar in &coroutine_layout.variant_fields[CoroutineLayout::UNRESUMED] { + internal_fields.swap_remove(upvar); + } let mut vars = vec![]; let bound_tys = tcx.mk_type_list_from_iter( - coroutine_layout - .as_ref() - .map_or_else(|| [].iter(), |l| l.field_tys.iter()) - .filter(|decl| !decl.ignore_for_traits) + internal_fields + .into_iter() + .filter_map(|saved_local| { + let saved_ty = &coroutine_layout.field_tys[saved_local]; + (!saved_ty.ignore_for_traits).then_some(saved_ty) + }) .map(|decl| { let ty = fold_regions(tcx, decl.ty, |re, debruijn| { assert_eq!(re, tcx.lifetimes.re_erased); diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index ac284653108cd..d0624a0ab23b8 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -11,6 +11,7 @@ use rustc_hir::attrs::AttributeKind; use rustc_hir::find_attr; use rustc_index::IndexVec; use rustc_middle::bug; +use rustc_middle::mir::CoroutineSavedLocal; use rustc_middle::query::Providers; use rustc_middle::traits::ObligationCause; use rustc_middle::ty::layout::{ @@ -20,6 +21,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{ self, AdtDef, CoroutineArgsExt, EarlyBinder, PseudoCanonicalInput, Ty, TyCtxt, TypeVisitableExt, }; +use rustc_session::config::PackCoroutineLayout; use rustc_session::{DataTypeKind, FieldInfo, FieldKind, SizeKind, VariantInfo}; use rustc_span::{Symbol, sym}; use tracing::{debug, instrument}; @@ -175,6 +177,7 @@ fn extract_const_value<'tcx>( } } +#[instrument(level = "debug", skip(cx), ret)] fn layout_of_uncached<'tcx>( cx: &LayoutCx<'tcx>, ty: Ty<'tcx>, @@ -515,8 +518,9 @@ fn layout_of_uncached<'tcx>( use rustc_middle::ty::layout::PrimitiveExt as _; let info = tcx.coroutine_layout(def_id, args)?; + debug!("info = {info:#?}"); - let local_layouts = info + let local_layouts: IndexVec<_, _> = info .field_tys .iter() .map(|local| { @@ -524,22 +528,31 @@ fn layout_of_uncached<'tcx>( let uninit_ty = Ty::new_maybe_uninit(tcx, field_ty.instantiate(tcx, args)); cx.spanned_layout_of(uninit_ty, local.source_info.span) }) - .try_collect::>()?; + .try_collect()?; - let prefix_layouts = args + let relocated_upvars = IndexVec::from_fn_n( + |local: CoroutineSavedLocal| info.relocated_upvars.get(&local).copied(), + info.field_tys.len(), + ); + let pack = match info.pack { + PackCoroutineLayout::No => rustc_abi::PackCoroutineLayout::Classic, + PackCoroutineLayout::CapturesOnly => rustc_abi::PackCoroutineLayout::CapturesOnly, + }; + let upvar_layouts = args .as_coroutine() .upvar_tys() .iter() .map(|ty| cx.layout_of(ty)) - .try_collect::>()?; - + .collect::>()?; let layout = cx .calc .coroutine( &local_layouts, - prefix_layouts, + &relocated_upvars, + upvar_layouts, &info.variant_fields, &info.storage_conflicts, + pack, |tag| TyAndLayout { ty: tag.primitive().to_ty(tcx), layout: tcx.mk_layout(LayoutData::scalar(cx, tag)), @@ -938,7 +951,11 @@ fn variant_info_for_coroutine<'tcx>( .then(|| Symbol::intern(&field_layout.ty.to_string())), } }) - .chain(upvar_fields.iter().copied()) + .chain( + if variant_idx == FIRST_VARIANT { &upvar_fields[..] } else { &[] } + .iter() + .copied(), + ) .collect(); // If the variant has no state-specific fields, then it's the size of the upvars. diff --git a/src/tools/clippy/tests/ui/explicit_write_in_test.stderr b/src/tools/clippy/tests/ui/explicit_write_in_test.stderr deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/tools/clippy/tests/ui/future_not_send.stderr b/src/tools/clippy/tests/ui/future_not_send.stderr index e366dc2d21958..8521eb13f62d7 100644 --- a/src/tools/clippy/tests/ui/future_not_send.stderr +++ b/src/tools/clippy/tests/ui/future_not_send.stderr @@ -93,10 +93,10 @@ LL | pub async fn public_future(&self) { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ future returned by `public_future` is not `Send` | note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> tests/ui/future_not_send.rs:49:32 + --> tests/ui/future_not_send.rs:49:33 | LL | pub async fn public_future(&self) { - | ^^^^^ has type `&Dummy` which is not `Send`, because `Dummy` is not `Sync` + | ^^^^ has type `&Dummy` which is not `Send`, because `Dummy` is not `Sync` = note: `std::rc::Rc<[u8]>` doesn't implement `std::marker::Sync` error: future cannot be sent between threads safely diff --git a/src/tools/clippy/tests/ui/needless_lifetimes.fixed b/src/tools/clippy/tests/ui/needless_lifetimes.fixed index 15ca409c95bd1..ef8a4a91677d0 100644 --- a/src/tools/clippy/tests/ui/needless_lifetimes.fixed +++ b/src/tools/clippy/tests/ui/needless_lifetimes.fixed @@ -3,6 +3,7 @@ #![warn(clippy::needless_lifetimes, clippy::elidable_lifetime_names)] #![allow( unused, + clippy::await_holding_lock, clippy::boxed_local, clippy::extra_unused_type_parameters, clippy::needless_pass_by_value, diff --git a/src/tools/clippy/tests/ui/needless_lifetimes.rs b/src/tools/clippy/tests/ui/needless_lifetimes.rs index af9649d729872..86da853dc238f 100644 --- a/src/tools/clippy/tests/ui/needless_lifetimes.rs +++ b/src/tools/clippy/tests/ui/needless_lifetimes.rs @@ -3,6 +3,7 @@ #![warn(clippy::needless_lifetimes, clippy::elidable_lifetime_names)] #![allow( unused, + clippy::await_holding_lock, clippy::boxed_local, clippy::extra_unused_type_parameters, clippy::needless_pass_by_value, diff --git a/src/tools/clippy/tests/ui/needless_lifetimes.stderr b/src/tools/clippy/tests/ui/needless_lifetimes.stderr index 138d0498c43e4..c7669e4d175b4 100644 --- a/src/tools/clippy/tests/ui/needless_lifetimes.stderr +++ b/src/tools/clippy/tests/ui/needless_lifetimes.stderr @@ -1,5 +1,5 @@ error: the following explicit lifetimes could be elided: 'a, 'b - --> tests/ui/needless_lifetimes.rs:19:23 + --> tests/ui/needless_lifetimes.rs:20:23 | LL | fn distinct_lifetimes<'a, 'b>(_x: &'a u8, _y: &'b u8, _z: u8) {} | ^^ ^^ ^^ ^^ @@ -13,7 +13,7 @@ LL + fn distinct_lifetimes(_x: &u8, _y: &u8, _z: u8) {} | error: the following explicit lifetimes could be elided: 'a, 'b - --> tests/ui/needless_lifetimes.rs:22:24 + --> tests/ui/needless_lifetimes.rs:23:24 | LL | fn distinct_and_static<'a, 'b>(_x: &'a u8, _y: &'b u8, _z: &'static u8) {} | ^^ ^^ ^^ ^^ @@ -25,7 +25,7 @@ LL + fn distinct_and_static(_x: &u8, _y: &u8, _z: &'static u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:33:15 + --> tests/ui/needless_lifetimes.rs:34:15 | LL | fn in_and_out<'a>(x: &'a u8, _y: u8) -> &'a u8 { | ^^ ^^ ^^ @@ -37,7 +37,7 @@ LL + fn in_and_out(x: &u8, _y: u8) -> &u8 { | error: the following explicit lifetimes could be elided: 'b - --> tests/ui/needless_lifetimes.rs:46:31 + --> tests/ui/needless_lifetimes.rs:47:31 | LL | fn multiple_in_and_out_2a<'a, 'b>(x: &'a u8, _y: &'b u8) -> &'a u8 { | ^^ ^^ @@ -49,7 +49,7 @@ LL + fn multiple_in_and_out_2a<'a>(x: &'a u8, _y: &u8) -> &'a u8 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:54:27 + --> tests/ui/needless_lifetimes.rs:55:27 | LL | fn multiple_in_and_out_2b<'a, 'b>(_x: &'a u8, y: &'b u8) -> &'b u8 { | ^^ ^^ @@ -61,7 +61,7 @@ LL + fn multiple_in_and_out_2b<'b>(_x: &u8, y: &'b u8) -> &'b u8 { | error: the following explicit lifetimes could be elided: 'b - --> tests/ui/needless_lifetimes.rs:72:26 + --> tests/ui/needless_lifetimes.rs:73:26 | LL | fn deep_reference_1a<'a, 'b>(x: &'a u8, _y: &'b u8) -> Result<&'a u8, ()> { | ^^ ^^ @@ -73,7 +73,7 @@ LL + fn deep_reference_1a<'a>(x: &'a u8, _y: &u8) -> Result<&'a u8, ()> { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:80:22 + --> tests/ui/needless_lifetimes.rs:81:22 | LL | fn deep_reference_1b<'a, 'b>(_x: &'a u8, y: &'b u8) -> Result<&'b u8, ()> { | ^^ ^^ @@ -85,7 +85,7 @@ LL + fn deep_reference_1b<'b>(_x: &u8, y: &'b u8) -> Result<&'b u8, ()> { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:90:21 + --> tests/ui/needless_lifetimes.rs:91:21 | LL | fn deep_reference_3<'a>(x: &'a u8, _y: u8) -> Result<&'a u8, ()> { | ^^ ^^ ^^ @@ -97,7 +97,7 @@ LL + fn deep_reference_3(x: &u8, _y: u8) -> Result<&u8, ()> { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:96:28 + --> tests/ui/needless_lifetimes.rs:97:28 | LL | fn where_clause_without_lt<'a, T>(x: &'a u8, _y: u8) -> Result<&'a u8, ()> | ^^ ^^ ^^ @@ -109,7 +109,7 @@ LL + fn where_clause_without_lt(x: &u8, _y: u8) -> Result<&u8, ()> | error: the following explicit lifetimes could be elided: 's - --> tests/ui/needless_lifetimes.rs:127:21 + --> tests/ui/needless_lifetimes.rs:128:21 | LL | fn self_and_out<'s>(&'s self) -> &'s u8 { | ^^ ^^ ^^ @@ -121,7 +121,7 @@ LL + fn self_and_out(&self) -> &u8 { | error: the following explicit lifetimes could be elided: 't - --> tests/ui/needless_lifetimes.rs:135:30 + --> tests/ui/needless_lifetimes.rs:136:30 | LL | fn self_and_in_out_1<'s, 't>(&'s self, _x: &'t u8) -> &'s u8 { | ^^ ^^ @@ -133,7 +133,7 @@ LL + fn self_and_in_out_1<'s>(&'s self, _x: &u8) -> &'s u8 { | error: the following explicit lifetimes could be elided: 's - --> tests/ui/needless_lifetimes.rs:143:26 + --> tests/ui/needless_lifetimes.rs:144:26 | LL | fn self_and_in_out_2<'s, 't>(&'s self, x: &'t u8) -> &'t u8 { | ^^ ^^ @@ -145,7 +145,7 @@ LL + fn self_and_in_out_2<'t>(&self, x: &'t u8) -> &'t u8 { | error: the following explicit lifetimes could be elided: 's, 't - --> tests/ui/needless_lifetimes.rs:148:29 + --> tests/ui/needless_lifetimes.rs:149:29 | LL | fn distinct_self_and_in<'s, 't>(&'s self, _x: &'t u8) {} | ^^ ^^ ^^ ^^ @@ -157,7 +157,7 @@ LL + fn distinct_self_and_in(&self, _x: &u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:172:21 + --> tests/ui/needless_lifetimes.rs:173:21 | LL | fn struct_with_lt4b<'a, 'b>(_foo: &'a Foo<'b>) -> &'b str { | ^^ ^^ @@ -169,7 +169,7 @@ LL + fn struct_with_lt4b<'b>(_foo: &Foo<'b>) -> &'b str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:188:22 + --> tests/ui/needless_lifetimes.rs:189:22 | LL | fn trait_obj_elided2<'a>(_arg: &'a dyn Drop) -> &'a str { | ^^ ^^ ^^ @@ -181,7 +181,7 @@ LL + fn trait_obj_elided2(_arg: &dyn Drop) -> &str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:199:20 + --> tests/ui/needless_lifetimes.rs:200:20 | LL | fn alias_with_lt4b<'a, 'b>(_foo: &'a FooAlias<'b>) -> &'b str { | ^^ ^^ @@ -193,7 +193,7 @@ LL + fn alias_with_lt4b<'b>(_foo: &FooAlias<'b>) -> &'b str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:204:30 + --> tests/ui/needless_lifetimes.rs:205:30 | LL | fn named_input_elided_output<'a>(_arg: &'a str) -> &str { | ^^ ^^ ^ @@ -205,7 +205,7 @@ LL + fn named_input_elided_output(_arg: &str) -> &str { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:213:19 + --> tests/ui/needless_lifetimes.rs:214:19 | LL | fn trait_bound_ok<'a, T: WithLifetime<'static>>(_: &'a u8, _: T) { | ^^ ^^ @@ -217,7 +217,7 @@ LL + fn trait_bound_ok>(_: &u8, _: T) { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:249:24 + --> tests/ui/needless_lifetimes.rs:250:24 | LL | fn needless_lt<'a>(x: &'a u8) {} | ^^ ^^ @@ -229,7 +229,7 @@ LL + fn needless_lt(x: &u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:254:24 + --> tests/ui/needless_lifetimes.rs:255:24 | LL | fn needless_lt<'a>(_x: &'a u8) {} | ^^ ^^ @@ -241,7 +241,7 @@ LL + fn needless_lt(_x: &u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:285:55 + --> tests/ui/needless_lifetimes.rs:286:55 | LL | fn impl_trait_elidable_nested_anonymous_lifetimes<'a>(i: &'a i32, f: impl Fn(&i32) -> &i32) -> &'a i32 { | ^^ ^^ ^^ @@ -253,7 +253,7 @@ LL + fn impl_trait_elidable_nested_anonymous_lifetimes(i: &i32, f: impl Fn(& | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:295:26 + --> tests/ui/needless_lifetimes.rs:296:26 | LL | fn generics_elidable<'a, T: Fn(&i32) -> &i32>(i: &'a i32, f: T) -> &'a i32 { | ^^ ^^ ^^ @@ -265,7 +265,7 @@ LL + fn generics_elidable &i32>(i: &i32, f: T) -> &i32 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:308:30 + --> tests/ui/needless_lifetimes.rs:309:30 | LL | fn where_clause_elidable<'a, T>(i: &'a i32, f: T) -> &'a i32 | ^^ ^^ ^^ @@ -277,7 +277,7 @@ LL + fn where_clause_elidable(i: &i32, f: T) -> &i32 | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:324:28 + --> tests/ui/needless_lifetimes.rs:325:28 | LL | fn pointer_fn_elidable<'a>(i: &'a i32, f: fn(&i32) -> &i32) -> &'a i32 { | ^^ ^^ ^^ @@ -289,7 +289,7 @@ LL + fn pointer_fn_elidable(i: &i32, f: fn(&i32) -> &i32) -> &i32 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:338:28 + --> tests/ui/needless_lifetimes.rs:339:28 | LL | fn nested_fn_pointer_3<'a>(_: &'a i32) -> fn(fn(&i32) -> &i32) -> i32 { | ^^ ^^ @@ -301,7 +301,7 @@ LL + fn nested_fn_pointer_3(_: &i32) -> fn(fn(&i32) -> &i32) -> i32 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:342:28 + --> tests/ui/needless_lifetimes.rs:343:28 | LL | fn nested_fn_pointer_4<'a>(_: &'a i32) -> impl Fn(fn(&i32)) { | ^^ ^^ @@ -313,7 +313,7 @@ LL + fn nested_fn_pointer_4(_: &i32) -> impl Fn(fn(&i32)) { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:365:21 + --> tests/ui/needless_lifetimes.rs:366:21 | LL | fn implicit<'a>(&'a self) -> &'a () { | ^^ ^^ ^^ @@ -325,7 +325,7 @@ LL + fn implicit(&self) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:369:25 + --> tests/ui/needless_lifetimes.rs:370:25 | LL | fn implicit_mut<'a>(&'a mut self) -> &'a () { | ^^ ^^ ^^ @@ -337,7 +337,7 @@ LL + fn implicit_mut(&mut self) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:374:21 + --> tests/ui/needless_lifetimes.rs:375:21 | LL | fn explicit<'a>(self: &'a Arc) -> &'a () { | ^^ ^^ ^^ @@ -349,7 +349,7 @@ LL + fn explicit(self: &Arc) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:379:25 + --> tests/ui/needless_lifetimes.rs:380:25 | LL | fn explicit_mut<'a>(self: &'a mut Rc) -> &'a () { | ^^ ^^ ^^ @@ -361,7 +361,7 @@ LL + fn explicit_mut(self: &mut Rc) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:392:31 + --> tests/ui/needless_lifetimes.rs:393:31 | LL | fn lifetime_elsewhere<'a>(self: Box, here: &'a ()) -> &'a () { | ^^ ^^ ^^ @@ -373,7 +373,7 @@ LL + fn lifetime_elsewhere(self: Box, here: &()) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:399:21 + --> tests/ui/needless_lifetimes.rs:400:21 | LL | fn implicit<'a>(&'a self) -> &'a (); | ^^ ^^ ^^ @@ -385,7 +385,7 @@ LL + fn implicit(&self) -> &(); | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:401:30 + --> tests/ui/needless_lifetimes.rs:402:30 | LL | fn implicit_provided<'a>(&'a self) -> &'a () { | ^^ ^^ ^^ @@ -397,7 +397,7 @@ LL + fn implicit_provided(&self) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:407:21 + --> tests/ui/needless_lifetimes.rs:408:21 | LL | fn explicit<'a>(self: &'a Arc) -> &'a (); | ^^ ^^ ^^ @@ -409,7 +409,7 @@ LL + fn explicit(self: &Arc) -> &(); | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:410:30 + --> tests/ui/needless_lifetimes.rs:411:30 | LL | fn explicit_provided<'a>(self: &'a Arc) -> &'a () { | ^^ ^^ ^^ @@ -421,7 +421,7 @@ LL + fn explicit_provided(self: &Arc) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:421:31 + --> tests/ui/needless_lifetimes.rs:422:31 | LL | fn lifetime_elsewhere<'a>(self: Box, here: &'a ()) -> &'a (); | ^^ ^^ ^^ @@ -433,7 +433,7 @@ LL + fn lifetime_elsewhere(self: Box, here: &()) -> &(); | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:423:40 + --> tests/ui/needless_lifetimes.rs:424:40 | LL | fn lifetime_elsewhere_provided<'a>(self: Box, here: &'a ()) -> &'a () { | ^^ ^^ ^^ @@ -445,7 +445,7 @@ LL + fn lifetime_elsewhere_provided(self: Box, here: &()) -> &() { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:433:12 + --> tests/ui/needless_lifetimes.rs:434:12 | LL | fn foo<'a>(x: &'a u8, y: &'_ u8) {} | ^^ ^^ @@ -457,7 +457,7 @@ LL + fn foo(x: &u8, y: &'_ u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:436:12 + --> tests/ui/needless_lifetimes.rs:437:12 | LL | fn bar<'a>(x: &'a u8, y: &'_ u8, z: &'_ u8) {} | ^^ ^^ @@ -469,7 +469,7 @@ LL + fn bar(x: &u8, y: &'_ u8, z: &'_ u8) {} | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:444:18 + --> tests/ui/needless_lifetimes.rs:445:18 | LL | fn one_input<'a>(x: &'a u8) -> &'a u8 { | ^^ ^^ ^^ @@ -481,7 +481,7 @@ LL + fn one_input(x: &u8) -> &u8 { | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:450:42 + --> tests/ui/needless_lifetimes.rs:451:42 | LL | fn multiple_inputs_output_not_elided<'a, 'b>(x: &'a u8, y: &'b u8, z: &'b u8) -> &'b u8 { | ^^ ^^ @@ -493,7 +493,7 @@ LL + fn multiple_inputs_output_not_elided<'b>(x: &u8, y: &'b u8, z: &'b u8) | error: the following explicit lifetimes could be elided: 'a - --> tests/ui/needless_lifetimes.rs:467:22 + --> tests/ui/needless_lifetimes.rs:468:22 | LL | fn one_input<'a>(x: &'a u8) -> &'a u8 { | ^^ ^^ ^^ diff --git a/src/tools/miri/tests/pass/async-drop.rs b/src/tools/miri/tests/pass/async-drop.rs index 4fa84384d9bdd..6754d4ffbef10 100644 --- a/src/tools/miri/tests/pass/async-drop.rs +++ b/src/tools/miri/tests/pass/async-drop.rs @@ -1,6 +1,8 @@ -//@revisions: stack tree -//@compile-flags: -Zmiri-strict-provenance -//@[tree]compile-flags: -Zmiri-tree-borrows +//@ revisions: stack tree stackrelocate treerelocate +//@ compile-flags: -Zmiri-strict-provenance -Zpack-coroutine-layout=no +//@ [tree] compile-flags: -Zmiri-tree-borrows +//@ [stackrelocate] compile-flags: -Zpack-coroutine-layout=captures-only +//@ [treerelocate] compile-flags: -Zmiri-tree-borrows -Zpack-coroutine-layout=captures-only // WARNING: If you would ever want to modify this test, // please consider modifying rustc's async drop test at diff --git a/src/tools/miri/tests/pass/async-drop.stackrelocate.stdout b/src/tools/miri/tests/pass/async-drop.stackrelocate.stdout new file mode 100644 index 0000000000000..fc53df2f1b485 --- /dev/null +++ b/src/tools/miri/tests/pass/async-drop.stackrelocate.stdout @@ -0,0 +1,23 @@ +AsyncInt::async_drop: 0 +AsyncInt::async_drop: 1 +AsyncInt::async_drop: 2 +AsyncInt::async_drop: 3 +AsyncInt::async_drop: 4 +AsyncStruct::async_drop: 6 +AsyncInt::async_drop: 7 +AsyncInt::async_drop: 8 +AsyncReference::async_drop: 10 +AsyncInt::async_drop: 11 +AsyncEnum(A)::async_drop: 12 +SyncInt::drop: 12 +AsyncEnum(B)::async_drop: 13 +AsyncInt::async_drop: 13 +SyncInt::drop: 14 +SyncThenAsync::drop: 15 +AsyncInt::async_drop: 16 +SyncInt::drop: 17 +AsyncInt::async_drop: 18 +AsyncInt::async_drop: 19 +AsyncInt::async_drop: 20 +AsyncUnion::async_drop: 21, 21 +AsyncInt::async_drop: 10 diff --git a/src/tools/miri/tests/pass/async-drop.treerelocate.stdout b/src/tools/miri/tests/pass/async-drop.treerelocate.stdout new file mode 100644 index 0000000000000..fc53df2f1b485 --- /dev/null +++ b/src/tools/miri/tests/pass/async-drop.treerelocate.stdout @@ -0,0 +1,23 @@ +AsyncInt::async_drop: 0 +AsyncInt::async_drop: 1 +AsyncInt::async_drop: 2 +AsyncInt::async_drop: 3 +AsyncInt::async_drop: 4 +AsyncStruct::async_drop: 6 +AsyncInt::async_drop: 7 +AsyncInt::async_drop: 8 +AsyncReference::async_drop: 10 +AsyncInt::async_drop: 11 +AsyncEnum(A)::async_drop: 12 +SyncInt::drop: 12 +AsyncEnum(B)::async_drop: 13 +AsyncInt::async_drop: 13 +SyncInt::drop: 14 +SyncThenAsync::drop: 15 +AsyncInt::async_drop: 16 +SyncInt::drop: 17 +AsyncInt::async_drop: 18 +AsyncInt::async_drop: 19 +AsyncInt::async_drop: 20 +AsyncUnion::async_drop: 21, 21 +AsyncInt::async_drop: 10 diff --git a/tests/codegen-llvm/async-fn-debug.rs b/tests/codegen-llvm/async-fn-debug.rs index ed704c7cc8b92..a1fc4f14db0b1 100644 --- a/tests/codegen-llvm/async-fn-debug.rs +++ b/tests/codegen-llvm/async-fn-debug.rs @@ -38,17 +38,17 @@ async fn async_fn_test() { // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) // CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "3", scope: [[VARIANT]], -// CHECK-SAME: file: [[FILE]], line: 13, +// CHECK-SAME: file: [[FILE]], line: 15, // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) -// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "4", scope: [[VARIANT]], -// CHECK-SAME: file: [[FILE]], line: 15, +// CHECK: [[S0:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "Suspend0", scope: [[GEN]], // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) -// CHECK: [[S1:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "Suspend1", scope: [[GEN]], +// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "s", scope: [[S0]] // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) -// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "s", scope: [[S1]] +// CHECK: {{!.*}} = !DIDerivedType(tag: DW_TAG_member, name: "4", scope: [[VARIANT]], +// CHECK-SAME: file: [[FILE]], line: 13, // CHECK-NOT: flags: DIFlagArtificial // CHECK-SAME: ) // CHECK: [[DISC]] = !DIDerivedType(tag: DW_TAG_member, name: "__state", scope: [[GEN]], diff --git a/tests/debuginfo/coroutine-objects.rs b/tests/debuginfo/coroutine-objects.rs index f1165de4bfaaf..f26702b0f9f1b 100644 --- a/tests/debuginfo/coroutine-objects.rs +++ b/tests/debuginfo/coroutine-objects.rs @@ -4,7 +4,7 @@ // with memory layout of discriminant for this particular enum // ensure that LLDB won't crash at least (like #57822). -//@ compile-flags:-g +//@ compile-flags: -g -Z pack-coroutine-layout=no //@ disable-gdb-pretty-printers //@ ignore-backends: gcc @@ -12,22 +12,22 @@ // gdb-command:run // gdb-command:print b -// gdb-check:$1 = coroutine_objects::main::{coroutine_env#0}::Unresumed{_ref__a: 0x[...]} +// gdb-check:$1 = coroutine_objects::main::{coroutine_env#0}::Unresumed{a: 0x[...]} // gdb-command:continue // gdb-command:print b -// gdb-check:$2 = coroutine_objects::main::{coroutine_env#0}::Suspend0{c: 6, d: 7, _ref__a: 0x[...]} +// gdb-check:$2 = coroutine_objects::main::{coroutine_env#0}::Suspend0{c: 6, d: 7} // gdb-command:continue // gdb-command:print b -// gdb-check:$3 = coroutine_objects::main::{coroutine_env#0}::Suspend1{c: 7, d: 8, _ref__a: 0x[...]} +// gdb-check:$3 = coroutine_objects::main::{coroutine_env#0}::Suspend1{c: 7, d: 8} // gdb-command:continue // gdb-command:print b -// gdb-check:$4 = coroutine_objects::main::{coroutine_env#0}::Returned{_ref__a: 0x[...]} +// gdb-check:$4 = coroutine_objects::main::{coroutine_env#0}::Returned // === LLDB TESTS ================================================================================== // lldb-command:run // lldb-command:v b -// lldb-check:(coroutine_objects::main::{coroutine_env#0}) b = { value = { _ref__a = 0x[...] } $discr$ = [...] } +// lldb-check:(coroutine_objects::main::{coroutine_env#0}) b = { value = { a = 0x[...] } $discr$ = [...] } // === CDB TESTS =================================================================================== diff --git a/tests/debuginfo/coroutine-objects_relocated.rs b/tests/debuginfo/coroutine-objects_relocated.rs new file mode 100644 index 0000000000000..cf5fde7ed30ca --- /dev/null +++ b/tests/debuginfo/coroutine-objects_relocated.rs @@ -0,0 +1,90 @@ +//@ min-lldb-version: 1800 +//@ ignore-gdb + +// LLDB (18.1+) now supports DW_TAG_variant_part, but there is some bug in either compiler or LLDB +// with memory layout of discriminant for this particular enum +// ensure that LLDB won't crash at least (like #57822). + +//@ compile-flags: -g -Z pack-coroutine-layout=captures-only +//@ disable-gdb-pretty-printers + +// === GDB TESTS =================================================================================== + +// gdb-command:run +// gdb-command:print b +// gdb-check:$1 = coroutine_objects_relocated::main::{coroutine_env#0}::Unresumed{a: 0x[...]} +// gdb-command:continue +// gdb-command:print b +// gdb-check:$2 = coroutine_objects_relocated::main::{coroutine_env#0}::Suspend0{c: 6, d: 7, a: 0x[...]} +// gdb-command:continue +// gdb-command:print b +// gdb-check:$3 = coroutine_objects_relocated::main::{coroutine_env#0}::Suspend1{c: 7, d: 8, a: 0x[...]} +// gdb-command:continue +// gdb-command:print b +// gdb-check:$4 = coroutine_objects_relocated::main::{coroutine_env#0}::Returned + +// === LLDB TESTS ================================================================================== + +// lldb-command:run +// lldb-command:v b +// lldb-check:(coroutine_objects_relocated::main::{coroutine_env#0}) b = { value = { a = 0x[...] } $discr$ = [...] } + +// === CDB TESTS =================================================================================== + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Unresumed [Type: enum2$] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 5 [Type: int *] + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Suspend0 [Type: enum2$] +// cdb-check: [+0x[...]] c : 6 [Type: int] +// cdb-check: [+0x[...]] d : 7 [Type: int] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 5 [Type: int *] + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Suspend1 [Type: enum2$] +// cdb-check: [+0x[...]] c : 7 [Type: int] +// cdb-check: [+0x[...]] d : 8 [Type: int] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 6 [Type: int *] + +// cdb-command: g +// cdb-command: dx b +// cdb-check: b : Returned [Type: enum2$] +// cdb-check: [+0x[...]] _ref__a : 0x[...] : 6 [Type: int *] + +#![feature(coroutines, coroutine_trait, stmt_expr_attributes)] + +use std::ops::Coroutine; +use std::pin::Pin; + +fn main() { + let mut a = 5; + let mut b = #[coroutine] + || { + let mut c = 6; + let mut d = 7; + + yield; + a += 1; + c += 1; + d += 1; + + yield; + println!("{} {} {}", a, c, d); + }; + _zzz(); // #break + Pin::new(&mut b).resume(()); + _zzz(); // #break + Pin::new(&mut b).resume(()); + _zzz(); // #break + Pin::new(&mut b).resume(()); + _zzz(); // #break +} + +#[inline(never)] +fn _zzz() { + () +} diff --git a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir index 435c1532895ca..b2bcc68428b07 100644 --- a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir +++ b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir @@ -2,7 +2,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> { debug _task_context => _2; - debug x => ((*_20).0: T); + debug x => (((*_20) as variant#0).0: T); let mut _0: std::task::Poll<()>; let _3: T; let mut _4: impl std::future::Future; @@ -83,7 +83,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) } bb11: { - drop(((*_20).0: T)) -> [return: bb10, unwind unreachable]; + drop((((*_20) as variant#0).0: T)) -> [return: bb10, unwind unreachable]; } bb12: { diff --git a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir index 1dc1d08136290..f0b94254fe26e 100644 --- a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir +++ b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir @@ -2,7 +2,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> { debug _task_context => _2; - debug x => ((*_20).0: T); + debug x => (((*_20) as variant#0).0: T); let mut _0: std::task::Poll<()>; let _3: T; let mut _4: impl std::future::Future; @@ -97,7 +97,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) } bb14: { - drop(((*_20).0: T)) -> [return: bb13, unwind: bb4]; + drop((((*_20) as variant#0).0: T)) -> [return: bb13, unwind: bb4]; } bb15 (cleanup): { diff --git a/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir index 6cad5b105d3e3..55fd5920e658f 100644 --- a/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir @@ -7,6 +7,7 @@ Panicked (2): [], }, storage_conflicts: BitMatrix(0x0) {}, + relocated_upvars: {}, } */ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> { diff --git a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir index 96ee37185db16..6ecd583035a56 100644 --- a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir @@ -40,13 +40,14 @@ Unresumed(0): [], Returned (1): [], Panicked (2): [], - Suspend0 (3): [_s0], - Suspend1 (4): [_s1], + Suspend0 (3): [_s1], + Suspend1 (4): [_s0], }, storage_conflicts: BitMatrix(2x2) { (_s0, _s0), (_s1, _s1), }, + relocated_upvars: {}, } */ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> Poll<()> { @@ -88,14 +89,14 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> let mut _38: u32; let mut _39: &mut {async fn body of b()}; scope 1 { - debug __awaitee => (((*_39) as variant#3).0: {async fn body of a()}); + debug __awaitee => (((*_39) as variant#4).0: {async fn body of a()}); let _17: (); scope 2 { debug result => _17; } } scope 3 { - debug __awaitee => (((*_39) as variant#4).0: {async fn body of a()}); + debug __awaitee => (((*_39) as variant#3).0: {async fn body of a()}); let _33: (); scope 4 { debug result => _33; @@ -123,7 +124,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_5); PlaceMention(_4); nop; - (((*_39) as variant#3).0: {async fn body of a()}) = move _4; + (((*_39) as variant#4).0: {async fn body of a()}) = move _4; goto -> bb4; } @@ -133,7 +134,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageLive(_10); StorageLive(_11); StorageLive(_12); - _12 = &mut (((*_39) as variant#3).0: {async fn body of a()}); + _12 = &mut (((*_39) as variant#4).0: {async fn body of a()}); _11 = &mut (*_12); _10 = Pin::<&mut {async fn body of a()}>::new_unchecked(move _11) -> [return: bb5, unwind unreachable]; } @@ -193,7 +194,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_12); StorageDead(_9); StorageDead(_8); - drop((((*_39) as variant#3).0: {async fn body of a()})) -> [return: bb12, unwind unreachable]; + drop((((*_39) as variant#4).0: {async fn body of a()})) -> [return: bb12, unwind unreachable]; } bb11: { @@ -225,7 +226,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_22); PlaceMention(_21); nop; - (((*_39) as variant#4).0: {async fn body of a()}) = move _21; + (((*_39) as variant#3).0: {async fn body of a()}) = move _21; goto -> bb16; } @@ -235,7 +236,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageLive(_26); StorageLive(_27); StorageLive(_28); - _28 = &mut (((*_39) as variant#4).0: {async fn body of a()}); + _28 = &mut (((*_39) as variant#3).0: {async fn body of a()}); _27 = &mut (*_28); _26 = Pin::<&mut {async fn body of a()}>::new_unchecked(move _27) -> [return: bb17, unwind unreachable]; } @@ -290,7 +291,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_28); StorageDead(_25); StorageDead(_24); - drop((((*_39) as variant#4).0: {async fn body of a()})) -> [return: bb23, unwind unreachable]; + drop((((*_39) as variant#3).0: {async fn body of a()})) -> [return: bb23, unwind unreachable]; } bb22: { diff --git a/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir b/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir index b61215dc28cb4..40f0facd5c48f 100644 --- a/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir +++ b/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir @@ -20,6 +20,7 @@ storage_conflicts: BitMatrix(1x1) { (_s0, _s0), }, + relocated_upvars: {}, } */ fn main::{closure#0}(_1: Pin<&mut {coroutine@$DIR/coroutine.rs:18:5: 18:18}>, _2: String) -> CoroutineState<(&str, String, &Location<'_>), ()> { diff --git a/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir b/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir index aac028a9e6c0e..07a5d7b4c89d2 100644 --- a/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir +++ b/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir @@ -20,6 +20,7 @@ storage_conflicts: BitMatrix(1x1) { (_s0, _s0), }, + relocated_upvars: {}, } */ fn main::{closure#1}(_1: Pin<&mut {coroutine@$DIR/coroutine.rs:25:5: 25:18}>, _2: String) -> CoroutineState<(&str, String, &Location<'_>), ()> { diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-abort.diff b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-abort.diff new file mode 100644 index 0000000000000..aec81b219d4a6 --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-abort.diff @@ -0,0 +1,91 @@ +- // MIR for `main::{closure#0}` before RelocateUpvars ++ // MIR for `main::{closure#0}` after RelocateUpvars + + fn main::{closure#0}(_1: {coroutine@$DIR/coroutine_relocate_upvars.rs:11:5: 11:7}, _2: ()) -> () + yields () + { +- debug x => (*(_1.0: &mut std::string::String)); ++ debug x => (*_9); ++ debug x => _9; + let mut _0: (); + let mut _3: std::string::String; + let _4: (); + let mut _5: (); + let mut _6: &mut std::string::String; + let mut _7: &mut std::string::String; + let mut _8: &mut std::string::String; ++ let mut _9: &mut std::string::String; ++ let mut _10: &mut std::string::String; + + bb0: { ++ StorageLive(_9); ++ StorageLive(_10); ++ _10 = move (_1.0: &mut std::string::String); ++ _9 = move _10; ++ StorageDead(_10); + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind unreachable]; + } + + bb1: { +- _6 = copy (_1.0: &mut std::string::String); ++ _6 = copy _9; + drop((*_6)) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- _7 = copy (_1.0: &mut std::string::String); ++ _7 = copy _9; + (*_7) = move _3; + goto -> bb3; + } + + bb3: { + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = (); + _4 = yield(move _5) -> [resume: bb4, drop: bb6]; + } + + bb4: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); +- drop(_1) -> [return: bb5, unwind unreachable]; ++ goto -> bb10; + } + + bb5: { ++ StorageDead(_9); + return; + } + + bb6: { + StorageDead(_5); + StorageDead(_4); + goto -> bb7; + } + + bb7: { +- drop(_1) -> [return: bb8, unwind unreachable]; ++ goto -> bb11; + } + + bb8: { + coroutine_drop; ++ } ++ ++ bb9 (cleanup): { ++ unreachable; ++ } ++ ++ bb10: { ++ goto -> bb5; ++ } ++ ++ bb11: { ++ goto -> bb8; + } + } + diff --git a/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-unwind.diff b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-unwind.diff new file mode 100644 index 0000000000000..3ed867e37ca70 --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.panic-unwind.diff @@ -0,0 +1,104 @@ +- // MIR for `main::{closure#0}` before RelocateUpvars ++ // MIR for `main::{closure#0}` after RelocateUpvars + + fn main::{closure#0}(_1: {coroutine@$DIR/coroutine_relocate_upvars.rs:11:5: 11:7}, _2: ()) -> () + yields () + { +- debug x => (*(_1.0: &mut std::string::String)); ++ debug x => (*_9); ++ debug x => _9; + let mut _0: (); + let mut _3: std::string::String; + let _4: (); + let mut _5: (); + let mut _6: &mut std::string::String; + let mut _7: &mut std::string::String; + let mut _8: &mut std::string::String; ++ let mut _9: &mut std::string::String; ++ let mut _10: &mut std::string::String; + + bb0: { ++ StorageLive(_9); ++ StorageLive(_10); ++ _10 = move (_1.0: &mut std::string::String); ++ _9 = move _10; ++ StorageDead(_10); + StorageLive(_3); + _3 = String::new() -> [return: bb1, unwind: bb10]; + } + + bb1: { +- _6 = copy (_1.0: &mut std::string::String); ++ _6 = copy _9; + drop((*_6)) -> [return: bb2, unwind: bb3]; + } + + bb2: { +- _7 = copy (_1.0: &mut std::string::String); ++ _7 = copy _9; + (*_7) = move _3; + goto -> bb4; + } + + bb3 (cleanup): { +- _8 = copy (_1.0: &mut std::string::String); ++ _8 = copy _9; + (*_8) = move _3; + goto -> bb10; + } + + bb4: { + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = (); + _4 = yield(move _5) -> [resume: bb5, drop: bb7]; + } + + bb5: { + StorageDead(_5); + StorageDead(_4); + _0 = const (); +- drop(_1) -> [return: bb6, unwind: bb11]; ++ goto -> bb12; + } + + bb6: { ++ StorageDead(_9); + return; + } + + bb7: { + StorageDead(_5); + StorageDead(_4); + goto -> bb8; + } + + bb8: { +- drop(_1) -> [return: bb9, unwind: bb11]; ++ goto -> bb13; + } + + bb9: { + coroutine_drop; + } + + bb10 (cleanup): { + StorageDead(_3); +- drop(_1) -> [return: bb11, unwind terminate(cleanup)]; ++ goto -> bb11; + } + + bb11 (cleanup): { + resume; ++ } ++ ++ bb12: { ++ goto -> bb6; ++ } ++ ++ bb13: { ++ goto -> bb9; + } + } + diff --git a/tests/mir-opt/coroutine_relocate_upvars.rs b/tests/mir-opt/coroutine_relocate_upvars.rs new file mode 100644 index 0000000000000..e65123d93e99c --- /dev/null +++ b/tests/mir-opt/coroutine_relocate_upvars.rs @@ -0,0 +1,21 @@ +// skip-filecheck +//@ compile-flags: -Zpack-coroutine-layout=captures-only +#![feature(coroutines, coroutine_trait, stmt_expr_attributes)] + +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY + +// EMIT_MIR coroutine_relocate_upvars.main-{closure#0}.RelocateUpvars.diff +fn main() { + let mut x = String::new(); + let capture_by_ref = #[coroutine] + || { + x = String::new(); + yield; + }; + let mut y = String::new(); + let capture_by_val = #[coroutine] + || { + y = String::new(); + yield; + }; +} diff --git a/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir index 222c7144ef07d..242afe469526b 100644 --- a/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir @@ -19,6 +19,7 @@ storage_conflicts: BitMatrix(1x1) { (_s0, _s0), }, + relocated_upvars: {}, } */ fn main::{closure#0}(_1: Pin<&mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13}>, _2: u8) -> CoroutineState<(), ()> { diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff index 4c1b25c786efc..5b5d99fcbedca 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff @@ -154,7 +154,7 @@ } + bb3: { -+ (((*_32) as variant#3).0: ActionPermit<'_, T>) = move ((*_32).0: ActionPermit<'_, T>); ++ (((*_32) as variant#3).0: ActionPermit<'_, T>) = move (((*_32) as variant#0).0: ActionPermit<'_, T>); + StorageLive(_12); + StorageLive(_13); + StorageLive(_14); diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff index 59417ce646513..0dc7973dddaa2 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff @@ -165,7 +165,7 @@ - StorageDead(_2); - return; + bb5: { -+ (((*_32) as variant#3).0: ActionPermit<'_, T>) = move ((*_32).0: ActionPermit<'_, T>); ++ (((*_32) as variant#3).0: ActionPermit<'_, T>) = move (((*_32) as variant#0).0: ActionPermit<'_, T>); + StorageLive(_12); + StorageLive(_13); + StorageLive(_14); diff --git a/tests/ui/async-await/async-drop/async-drop-initial.run.stdout b/tests/ui/async-await/async-drop/async-drop-initial.classic.run.stdout similarity index 94% rename from tests/ui/async-await/async-drop/async-drop-initial.run.stdout rename to tests/ui/async-await/async-drop/async-drop-initial.classic.run.stdout index 9cae4331caf92..792880d835935 100644 --- a/tests/ui/async-await/async-drop/async-drop-initial.run.stdout +++ b/tests/ui/async-await/async-drop/async-drop-initial.classic.run.stdout @@ -20,3 +20,4 @@ AsyncInt::Dropper::poll: 18 AsyncInt::Dropper::poll: 19 AsyncInt::Dropper::poll: 20 AsyncUnion::Dropper::poll: 21, 21 +AsyncUnion::Dropper::poll: 21, 21 diff --git a/tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout b/tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout new file mode 100644 index 0000000000000..792880d835935 --- /dev/null +++ b/tests/ui/async-await/async-drop/async-drop-initial.relocate.run.stdout @@ -0,0 +1,23 @@ +AsyncInt::Dropper::poll: 0 +AsyncInt::Dropper::poll: 1 +AsyncInt::Dropper::poll: 2 +AsyncInt::Dropper::poll: 3 +AsyncInt::Dropper::poll: 4 +AsyncStruct::Dropper::poll: 6 +AsyncInt::Dropper::poll: 7 +AsyncInt::Dropper::poll: 8 +AsyncReference::Dropper::poll: 10 +AsyncInt::Dropper::poll: 11 +AsyncEnum(A)::Dropper::poll: 12 +SyncInt::drop: 12 +AsyncEnum(B)::Dropper::poll: 13 +AsyncInt::Dropper::poll: 13 +SyncInt::drop: 14 +SyncThenAsync::drop: 15 +AsyncInt::Dropper::poll: 16 +SyncInt::drop: 17 +AsyncInt::Dropper::poll: 18 +AsyncInt::Dropper::poll: 19 +AsyncInt::Dropper::poll: 20 +AsyncUnion::Dropper::poll: 21, 21 +AsyncUnion::Dropper::poll: 21, 21 diff --git a/tests/ui/async-await/async-drop/async-drop-initial.rs b/tests/ui/async-await/async-drop/async-drop-initial.rs index cd33c143fba01..ba76d0e951f25 100644 --- a/tests/ui/async-await/async-drop/async-drop-initial.rs +++ b/tests/ui/async-await/async-drop/async-drop-initial.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ run-pass //@ check-run-results @@ -11,26 +14,32 @@ //@ edition: 2021 // FIXME(zetanumbers): consider AsyncDestruct::async_drop cleanup tests -use core::future::{async_drop_in_place, AsyncDrop, Future}; +use core::future::{AsyncDrop, Future, async_drop_in_place}; use core::hint::black_box; use core::mem::{self, ManuallyDrop}; -use core::pin::{pin, Pin}; +use core::pin::{Pin, pin}; +use core::sync::atomic::{AtomicBool, Ordering}; use core::task::{Context, Poll, Waker}; -async fn test_async_drop(x: T, _size: usize) { +static PASS: AtomicBool = AtomicBool::new(false); + +async fn test_async_drop(x: T, #[allow(unused)] expect: usize) { let mut x = mem::MaybeUninit::new(x); let dtor = pin!(unsafe { async_drop_in_place(x.as_mut_ptr()) }); // FIXME(zetanumbers): This check fully depends on the layout of // the coroutine state, since async destructor combinators are just // async functions. + #[allow(unused)] + let got = mem::size_of_val(&*dtor); #[cfg(target_pointer_width = "64")] - assert_eq!( - mem::size_of_val(&*dtor), - _size, - "sizes did not match for async destructor of type {}", - core::any::type_name::(), - ); + if expect != got { + println!( + "sizes did not match for async destructor of type {}, expect {expect}, got {got}", + core::any::type_name::() + ); + PASS.store(false, Ordering::Relaxed); + } test_idempotency(dtor).await; } @@ -47,6 +56,7 @@ where } fn main() { + PASS.store(true, Ordering::Relaxed); let waker = Waker::noop(); let mut cx = Context::from_waker(&waker); @@ -54,16 +64,17 @@ fn main() { let fut = pin!(async { test_async_drop(Int(0), 16).await; test_async_drop(AsyncInt(0), 32).await; - test_async_drop([AsyncInt(1), AsyncInt(2)], 104).await; - test_async_drop((AsyncInt(3), AsyncInt(4)), 120).await; + test_async_drop([AsyncInt(1), AsyncInt(2)], [96, 104][cfg!(classic) as usize]).await; + test_async_drop((AsyncInt(3), AsyncInt(4)), [112, 120][cfg!(classic) as usize]).await; test_async_drop(5, 16).await; let j = 42; test_async_drop(&i, 16).await; test_async_drop(&j, 16).await; test_async_drop( AsyncStruct { b: AsyncInt(8), a: AsyncInt(7), i: 6 }, - 136, - ).await; + if cfg!(panic = "unwind") { [128, 136][cfg!(classic) as usize] } else { 136 }, + ) + .await; test_async_drop(ManuallyDrop::new(AsyncInt(9)), 16).await; let foo = AsyncInt(10); @@ -81,13 +92,13 @@ fn main() { ) .await; - test_async_drop(AsyncEnum::A(AsyncInt(12)), 104).await; - test_async_drop(AsyncEnum::B(SyncInt(13)), 104).await; + test_async_drop(AsyncEnum::A(AsyncInt(12)), [96, 104][cfg!(classic) as usize]).await; + test_async_drop(AsyncEnum::B(SyncInt(13)), [96, 104][cfg!(classic) as usize]).await; test_async_drop(SyncInt(14), 16).await; test_async_drop( SyncThenAsync { i: 15, a: AsyncInt(16), b: SyncInt(17), c: AsyncInt(18) }, - 120, + [112, 120][cfg!(classic) as usize], ) .await; @@ -109,9 +120,11 @@ fn main() { .await; test_async_drop(AsyncUnion { signed: 21 }, 32).await; + test_async_drop(AsyncUnion { signed: 21 }, 32).await; }); let res = fut.poll(&mut cx); assert_eq!(res, Poll::Ready(())); + assert!(PASS.load(Ordering::Relaxed)); } struct AsyncInt(i32); @@ -228,19 +241,13 @@ union AsyncUnion { impl Drop for AsyncUnion { fn drop(&mut self) { - println!( - "AsyncUnion::drop: {}, {}", - unsafe { self.signed }, - unsafe { self.unsigned }, - ); + println!("AsyncUnion::drop: {}, {}", unsafe { self.signed }, unsafe { self.unsigned },); } } impl AsyncDrop for AsyncUnion { async fn drop(self: Pin<&mut Self>) { - println!( - "AsyncUnion::Dropper::poll: {}, {}", - unsafe { self.signed }, - unsafe { self.unsigned }, - ); + println!("AsyncUnion::Dropper::poll: {}, {}", unsafe { self.signed }, unsafe { + self.unsigned + },); } } diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout b/tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout similarity index 88% rename from tests/ui/async-await/future-sizes/async-awaiting-fut.stdout rename to tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout index b30c15bcbe6ed..c8363ec90e03e 100644 --- a/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.classic.stdout @@ -14,26 +14,24 @@ print-type-size field `.value`: 3077 bytes print-type-size type: `{async fn body of calls_fut<{async fn body of big_fut()}>()}`: 3077 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes -print-type-size variant `Suspend0`: 2052 bytes -print-type-size upvar `.fut`: 1025 bytes print-type-size local `.fut`: 1025 bytes +print-type-size upvar `.fut`: 1025 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 2052 bytes +print-type-size padding: 1025 bytes +print-type-size local `.fut`: 1025 bytes, alignment: 1 bytes print-type-size local `..coroutine_field4`: 1 bytes, type: bool print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} print-type-size variant `Suspend1`: 3076 bytes -print-type-size upvar `.fut`: 1025 bytes -print-type-size padding: 1025 bytes +print-type-size padding: 2050 bytes print-type-size local `..coroutine_field4`: 1 bytes, alignment: 1 bytes, type: bool print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of big_fut()} print-type-size variant `Suspend2`: 2052 bytes -print-type-size upvar `.fut`: 1025 bytes -print-type-size local `.fut`: 1025 bytes +print-type-size padding: 1025 bytes +print-type-size local `.fut`: 1025 bytes, alignment: 1 bytes print-type-size local `..coroutine_field4`: 1 bytes, type: bool print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} print-type-size variant `Returned`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes print-type-size variant `Panicked`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes print-type-size field `.value`: 1025 bytes print-type-size type: `std::mem::MaybeUninit<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes @@ -43,11 +41,16 @@ print-type-size field `.value`: 1025 bytes print-type-size type: `{async fn body of big_fut()}`: 1025 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes +print-type-size local `.arg`: 1024 bytes +print-type-size upvar `.arg`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes print-type-size field `.waker`: 8 bytes print-type-size field `.local_waker`: 8 bytes diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout b/tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout new file mode 100644 index 0000000000000..2c658c5f925a0 --- /dev/null +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.relocate.stdout @@ -0,0 +1,111 @@ +print-type-size type: `{async fn body of test()}`: 2053 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Suspend0`: 2052 bytes +print-type-size local `.__awaitee`: 2052 bytes, type: {async fn body of calls_fut<{async fn body of big_fut()}>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 2052 bytes, alignment: 1 bytes +print-type-size field `.value`: 2052 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 2052 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 2052 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 2052 bytes +print-type-size type: `{async fn body of calls_fut<{async fn body of big_fut()}>()}`: 2052 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 2051 bytes +print-type-size upvar `.fut`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size padding: 1026 bytes +print-type-size local `.fut`: 1025 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 1027 bytes +print-type-size local `.fut`: 1025 bytes +print-type-size local `..coroutine_field4`: 1 bytes, type: bool +print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} +print-type-size variant `Suspend1`: 2051 bytes +print-type-size padding: 1025 bytes +print-type-size local `..coroutine_field4`: 1 bytes, alignment: 1 bytes, type: bool +print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of big_fut()} +print-type-size variant `Suspend2`: 1027 bytes +print-type-size local `.fut`: 1025 bytes +print-type-size local `..coroutine_field4`: 1 bytes, type: bool +print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes +print-type-size field `.value`: 1025 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1025 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1025 bytes +print-type-size type: `{async fn body of big_fut()}`: 1025 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size upvar `.arg`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.arg`: 1024 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes +print-type-size field `.waker`: 8 bytes +print-type-size field `.local_waker`: 8 bytes +print-type-size field `.ext`: 16 bytes +print-type-size field `._marker`: 0 bytes +print-type-size field `._marker2`: 0 bytes +print-type-size type: `std::panic::Location<'_>`: 24 bytes, alignment: 8 bytes +print-type-size field `.filename`: 16 bytes +print-type-size field `.line`: 4 bytes +print-type-size field `.col`: 4 bytes +print-type-size field `._filename`: 0 bytes +print-type-size type: `core::task::wake::ExtData<'_>`: 16 bytes, alignment: 8 bytes +print-type-size variant `Some`: 16 bytes +print-type-size field `.0`: 16 bytes +print-type-size variant `None`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size type: `std::panic::AssertUnwindSafe>`: 16 bytes, alignment: 8 bytes +print-type-size field `.0`: 16 bytes +print-type-size type: `std::ptr::NonNull`: 16 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 16 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of big_fut()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of test()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of wait()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::ptr::DynMetadata`: 8 bytes, alignment: 8 bytes +print-type-size field `._vtable_ptr`: 8 bytes +print-type-size field `._phantom`: 0 bytes +print-type-size type: `std::ptr::NonNull`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::mem::ManuallyDrop`: 1 bytes, alignment: 1 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::MaybeUninit`: 1 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::task::Poll<()>`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `{async fn body of wait()}`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::marker::PhantomData<&str>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData<*mut ()>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData &()>`: 0 bytes, alignment: 1 bytes diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.rs b/tests/ui/async-await/future-sizes/async-awaiting-fut.rs index 7113f591630d1..d646c7b8f039a 100644 --- a/tests/ui/async-await/future-sizes/async-awaiting-fut.rs +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only // FIXME(#61117): Respect debuginfo-level-tests, do not force debuginfo=0 //@ compile-flags: -C debuginfo=0 //@ compile-flags: -C panic=abort -Z print-type-sizes --crate-type lib diff --git a/tests/ui/async-await/future-sizes/future-as-arg.rs b/tests/ui/async-await/future-sizes/future-as-arg.rs index 317c17b9b3e59..3bbbadc2099df 100644 --- a/tests/ui/async-await/future-sizes/future-as-arg.rs +++ b/tests/ui/async-await/future-sizes/future-as-arg.rs @@ -8,9 +8,10 @@ async fn use_future(fut: impl std::future::Future) { } fn main() { - let actual = std::mem::size_of_val( - &use_future(use_future(use_future(use_future(use_future(test([0; 16]))))))); + let actual = std::mem::size_of_val(&use_future(use_future(use_future(use_future( + use_future(test([0; 16])), + ))))); // Not using an exact number in case it slightly changes over different commits - let expected = 550; - assert!(actual > expected, "expected: >{expected}, actual: {actual}"); + let expected = 21; + assert!(actual >= expected, "expected: >{expected}, actual: {actual}"); } diff --git a/tests/ui/async-await/future-sizes/large-arg.stdout b/tests/ui/async-await/future-sizes/large-arg.classic.stdout similarity index 83% rename from tests/ui/async-await/future-sizes/large-arg.stdout rename to tests/ui/async-await/future-sizes/large-arg.classic.stdout index e00420d1493f4..37bc7dc658191 100644 --- a/tests/ui/async-await/future-sizes/large-arg.stdout +++ b/tests/ui/async-await/future-sizes/large-arg.classic.stdout @@ -14,14 +14,13 @@ print-type-size field `.value`: 3075 bytes print-type-size type: `{async fn body of a<[u8; 1024]>()}`: 3075 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Suspend0`: 3074 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size local `.__awaitee`: 2050 bytes, type: {async fn body of b<[u8; 1024]>()} +print-type-size padding: 1024 bytes +print-type-size local `.__awaitee`: 2050 bytes, alignment: 1 bytes, type: {async fn body of b<[u8; 1024]>()} print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of b<[u8; 1024]>()}>`: 2050 bytes, alignment: 1 bytes print-type-size field `.value`: 2050 bytes print-type-size type: `std::mem::MaybeUninit<{async fn body of b<[u8; 1024]>()}>`: 2050 bytes, alignment: 1 bytes @@ -31,14 +30,13 @@ print-type-size field `.value`: 2050 bytes print-type-size type: `{async fn body of b<[u8; 1024]>()}`: 2050 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Suspend0`: 2049 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of c<[u8; 1024]>()} +print-type-size padding: 1024 bytes +print-type-size local `.__awaitee`: 1025 bytes, alignment: 1 bytes, type: {async fn body of c<[u8; 1024]>()} print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes print-type-size field `.value`: 1025 bytes print-type-size type: `std::mem::MaybeUninit<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes @@ -53,11 +51,16 @@ print-type-size variant `Pending`: 0 bytes print-type-size type: `{async fn body of c<[u8; 1024]>()}`: 1025 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes print-type-size field `.waker`: 8 bytes print-type-size field `.local_waker`: 8 bytes diff --git a/tests/ui/async-await/future-sizes/large-arg.relocate.stdout b/tests/ui/async-await/future-sizes/large-arg.relocate.stdout new file mode 100644 index 0000000000000..53465757a3d1e --- /dev/null +++ b/tests/ui/async-await/future-sizes/large-arg.relocate.stdout @@ -0,0 +1,103 @@ +print-type-size type: `{async fn body of test()}`: 1028 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Suspend0`: 1027 bytes +print-type-size local `.__awaitee`: 1027 bytes, type: {async fn body of a<[u8; 1024]>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of a<[u8; 1024]>()}>`: 1027 bytes, alignment: 1 bytes +print-type-size field `.value`: 1027 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of a<[u8; 1024]>()}>`: 1027 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1027 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1027 bytes +print-type-size type: `{async fn body of a<[u8; 1024]>()}`: 1027 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size upvar `.t`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.t`: 1024 bytes +print-type-size variant `Suspend0`: 1026 bytes +print-type-size local `.__awaitee`: 1026 bytes, type: {async fn body of b<[u8; 1024]>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of b<[u8; 1024]>()}>`: 1026 bytes, alignment: 1 bytes +print-type-size field `.value`: 1026 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of b<[u8; 1024]>()}>`: 1026 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1026 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1026 bytes +print-type-size type: `{async fn body of b<[u8; 1024]>()}`: 1026 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size upvar `.t`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.t`: 1024 bytes +print-type-size variant `Suspend0`: 1025 bytes +print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of c<[u8; 1024]>()} +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes +print-type-size field `.value`: 1025 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1025 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1025 bytes +print-type-size type: `std::task::Poll<[u8; 1024]>`: 1025 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 1024 bytes +print-type-size field `.0`: 1024 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `{async fn body of c<[u8; 1024]>()}`: 1025 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 1024 bytes +print-type-size upvar `.t`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.t`: 1024 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes +print-type-size field `.waker`: 8 bytes +print-type-size field `.local_waker`: 8 bytes +print-type-size field `.ext`: 16 bytes +print-type-size field `._marker`: 0 bytes +print-type-size field `._marker2`: 0 bytes +print-type-size type: `std::panic::Location<'_>`: 24 bytes, alignment: 8 bytes +print-type-size field `.filename`: 16 bytes +print-type-size field `.line`: 4 bytes +print-type-size field `.col`: 4 bytes +print-type-size field `._filename`: 0 bytes +print-type-size type: `core::task::wake::ExtData<'_>`: 16 bytes, alignment: 8 bytes +print-type-size variant `Some`: 16 bytes +print-type-size field `.0`: 16 bytes +print-type-size variant `None`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size type: `std::panic::AssertUnwindSafe>`: 16 bytes, alignment: 8 bytes +print-type-size field `.0`: 16 bytes +print-type-size type: `std::ptr::NonNull`: 16 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 16 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of a<[u8; 1024]>()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of b<[u8; 1024]>()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of c<[u8; 1024]>()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of test()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::ptr::DynMetadata`: 8 bytes, alignment: 8 bytes +print-type-size field `._vtable_ptr`: 8 bytes +print-type-size field `._phantom`: 0 bytes +print-type-size type: `std::ptr::NonNull`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::task::Poll<()>`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `std::marker::PhantomData<&str>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData<*mut ()>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData &()>`: 0 bytes, alignment: 1 bytes diff --git a/tests/ui/async-await/future-sizes/large-arg.rs b/tests/ui/async-await/future-sizes/large-arg.rs index b05a2b7191512..2dd440b3f3a7f 100644 --- a/tests/ui/async-await/future-sizes/large-arg.rs +++ b/tests/ui/async-await/future-sizes/large-arg.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only // FIXME(#61117): Respect debuginfo-level-tests, do not force debuginfo=0 //@ compile-flags: -C debuginfo=0 //@ compile-flags: -C panic=abort -Z print-type-sizes --crate-type=lib diff --git a/tests/ui/async-await/issue-70818.stderr b/tests/ui/async-await/issue-70818.stderr index 07fd20cdd77ea..efc412cef098b 100644 --- a/tests/ui/async-await/issue-70818.stderr +++ b/tests/ui/async-await/issue-70818.stderr @@ -5,10 +5,10 @@ LL | async { (ty, ty1) } | ^^^^^^^^^^^^^^^^^^^ future created by async block is not `Send` | note: captured value is not `Send` - --> $DIR/issue-70818.rs:6:18 + --> $DIR/issue-70818.rs:4:27 | -LL | async { (ty, ty1) } - | ^^^ has type `U` which is not `Send` +LL | fn foo(ty: T, ty1: U) -> impl Future + Send { + | ^^^ has type `U` which is not `Send` note: required by a bound in an opaque type --> $DIR/issue-70818.rs:4:69 | @@ -26,10 +26,10 @@ LL | fn foo(ty: T, ty1: U) -> impl Future + Send { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ future created by async block is not `Send` | note: captured value is not `Send` - --> $DIR/issue-70818.rs:6:18 + --> $DIR/issue-70818.rs:4:27 | -LL | async { (ty, ty1) } - | ^^^ has type `U` which is not `Send` +LL | fn foo(ty: T, ty1: U) -> impl Future + Send { + | ^^^ has type `U` which is not `Send` help: consider restricting type parameter `U` with trait `Send` | LL | fn foo(ty: T, ty1: U) -> impl Future + Send { diff --git a/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs b/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs index f6c9fdd6d6806..63adb5a725172 100644 --- a/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs +++ b/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs @@ -9,7 +9,7 @@ pub async fn async_fn(x: &mut i32) -> &i32 { y } -pub fn async_closure(x: &mut i32) -> impl Future { +pub fn async_closure(x: &mut i32) -> impl Future { (async move || { //~^ ERROR lifetime may not live long enough //~| ERROR temporary value dropped while borrowed @@ -19,7 +19,7 @@ pub fn async_closure(x: &mut i32) -> impl Future { })() } -pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { +pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { (async move || -> &i32 { //~^ ERROR lifetime may not live long enough //~| ERROR temporary value dropped while borrowed @@ -29,7 +29,7 @@ pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future impl Future { +pub fn async_block(x: &mut i32) -> impl Future { async move { let y = &*x; *x += 1; //~ ERROR cannot assign to `*x` because it is borrowed diff --git a/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr b/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr index e1f268116fc54..cb559898c2fc5 100644 --- a/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr +++ b/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr @@ -44,7 +44,7 @@ LL | | })() error[E0716]: temporary value dropped while borrowed --> $DIR/issue-74072-lifetime-name-annotations.rs:13:5 | -LL | pub fn async_closure(x: &mut i32) -> impl Future { +LL | pub fn async_closure(x: &mut i32) -> impl Future { | - let's call the lifetime of this reference `'1` LL | // (async move || { LL | || @@ -93,7 +93,7 @@ LL | | })() error[E0716]: temporary value dropped while borrowed --> $DIR/issue-74072-lifetime-name-annotations.rs:23:5 | -LL | pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { +LL | pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { | - let's call the lifetime of this reference `'1` LL | // (async move || -> &i32 { LL | || diff --git a/tests/ui/async-await/issue-86507.rs b/tests/ui/async-await/issue-86507.rs index 484122a1ddcfd..6dbebc9328677 100644 --- a/tests/ui/async-await/issue-86507.rs +++ b/tests/ui/async-await/issue-86507.rs @@ -12,9 +12,14 @@ trait Foo { impl Foo for () { fn bar<'me, 'async_trait, T: Send>(x: &'me T) + //~^ NOTE captured value is not `Send` + //~| NOTE has type `&T` which is not `Send` -> Pin + Send + 'async_trait>> where 'me:'async_trait { - Box::pin( //~ ERROR future cannot be sent between threads safely + Box::pin( + //~^ ERROR future cannot be sent between threads safely + //~| NOTE future created by async block is not `Send` + //~| NOTE required for the cast from async move { let x = x; } diff --git a/tests/ui/async-await/issue-86507.stderr b/tests/ui/async-await/issue-86507.stderr index c71801dcfc854..32b9f795ee79d 100644 --- a/tests/ui/async-await/issue-86507.stderr +++ b/tests/ui/async-await/issue-86507.stderr @@ -1,19 +1,17 @@ error: future cannot be sent between threads safely - --> $DIR/issue-86507.rs:17:13 + --> $DIR/issue-86507.rs:19:13 | LL | / Box::pin( -LL | | async move { -LL | | let x = x; -LL | | } +... | LL | | ) | |_____________^ future created by async block is not `Send` | note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> $DIR/issue-86507.rs:19:29 + --> $DIR/issue-86507.rs:14:40 | -LL | let x = x; - | ^ has type `&T` which is not `Send`, because `T` is not `Sync` - = note: required for the cast from `Pin>` to `Pin + Send>>` +LL | fn bar<'me, 'async_trait, T: Send>(x: &'me T) + | ^ has type `&T` which is not `Send`, because `T` is not `Sync` + = note: required for the cast from `Pin>` to `Pin + Send>>` help: consider further restricting type parameter `T` with trait `Sync` | LL | fn bar<'me, 'async_trait, T: Send + std::marker::Sync>(x: &'me T) diff --git a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.rs b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.rs index 9317ab5cd2756..d5e849b77e812 100644 --- a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.rs +++ b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.rs @@ -2,12 +2,25 @@ //@ edition: 2018 //@ compile-flags: --target thumbv8m.main-none-eabi --crate-type lib //@ needs-llvm-components: arm -#![feature(cmse_nonsecure_entry, c_variadic, no_core, lang_items)] +#![feature(cmse_nonsecure_entry, rustc_attrs, c_variadic, no_core, lang_items, transparent_unions)] #![no_core] extern crate minicore; use minicore::*; +#[lang = "clone"] +pub trait Clone: Sized {} + +#[lang = "maybe_uninit"] +#[repr(transparent)] +#[rustc_pub_transparent] +pub union MaybeUninit { + uninit: (), + value: ManuallyDrop, +} +impl Copy for MaybeUninit {} +impl Clone for MaybeUninit {} + #[lang = "va_list"] struct VaList(*mut u8); diff --git a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.stderr b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.stderr index 948f8f5747b0e..060dc4129de2d 100644 --- a/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.stderr +++ b/tests/ui/cmse-nonsecure/cmse-nonsecure-entry/c-variadic.stderr @@ -1,5 +1,5 @@ error: `...` is not supported for `extern "cmse-nonsecure-entry"` functions - --> $DIR/c-variadic.rs:14:60 + --> $DIR/c-variadic.rs:27:60 | LL | unsafe extern "cmse-nonsecure-entry" fn c_variadic(_: u32, _: ...) { | ----------------------------- ^^^^^^ @@ -9,13 +9,13 @@ LL | unsafe extern "cmse-nonsecure-entry" fn c_variadic(_: u32, _: ...) { = help: only `extern "C"` and `extern "C-unwind"` functions may have a C variable argument list error: functions cannot be both `async` and C-variadic - --> $DIR/c-variadic.rs:19:1 + --> $DIR/c-variadic.rs:32:1 | LL | async unsafe extern "cmse-nonsecure-entry" fn async_and_c_variadic(_: ...) { | ^^^^^ `async` because of this ^^^^^^ C-variadic because of this error: `...` is not supported for `extern "cmse-nonsecure-entry"` functions - --> $DIR/c-variadic.rs:19:68 + --> $DIR/c-variadic.rs:32:68 | LL | async unsafe extern "cmse-nonsecure-entry" fn async_and_c_variadic(_: ...) { | ----------------------------- ^^^^^^ diff --git a/tests/ui/coroutine/clone-impl.stderr b/tests/ui/coroutine/clone-impl.classic.stderr similarity index 58% rename from tests/ui/coroutine/clone-impl.stderr rename to tests/ui/coroutine/clone-impl.classic.stderr index f316902a42d08..73513cf8df612 100644 --- a/tests/ui/coroutine/clone-impl.stderr +++ b/tests/ui/coroutine/clone-impl.classic.stderr @@ -1,81 +1,81 @@ -error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:44:5: 44:12}` - --> $DIR/clone-impl.rs:48:5 +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` + --> $DIR/clone-impl.rs:51:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:44:5: 44:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` ... LL | check_copy(&gen_clone_0); - | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:44:5: 44:12}`, the trait `Copy` is not implemented for `Vec` + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}`, the trait `Copy` is not implemented for `Vec` | note: captured value does not implement `Copy` - --> $DIR/clone-impl.rs:46:14 + --> $DIR/clone-impl.rs:45:9 | -LL | drop(clonable_0); - | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +LL | let clonable_0: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` -error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:55:5: 55:12}` - --> $DIR/clone-impl.rs:60:5 +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` + --> $DIR/clone-impl.rs:63:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:55:5: 55:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` ... LL | check_copy(&gen_clone_1); - | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:55:5: 55:12}`, the trait `Copy` is not implemented for `Vec` + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}`, the trait `Copy` is not implemented for `Vec` | note: coroutine does not implement `Copy` as this value is used across a yield - --> $DIR/clone-impl.rs:57:9 + --> $DIR/clone-impl.rs:60:9 | LL | let v = vec!['a']; | - has type `Vec` which does not implement `Copy` LL | yield; | ^^^^^ yield occurs here, with `v` maybe used later note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` -error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:68:5: 68:12}` - --> $DIR/clone-impl.rs:74:5 +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` + --> $DIR/clone-impl.rs:77:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:68:5: 68:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` ... LL | check_copy(&gen_clone_1); - | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:68:5: 68:12}`, the trait `Copy` is not implemented for `Vec` + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}`, the trait `Copy` is not implemented for `Vec` | note: captured value does not implement `Copy` - --> $DIR/clone-impl.rs:72:14 + --> $DIR/clone-impl.rs:69:9 | -LL | drop(clonable_1); - | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +LL | let clonable_1: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` -error[E0277]: the trait bound `NonClone: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` - --> $DIR/clone-impl.rs:86:5 +error[E0277]: the trait bound `NonClone: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:89:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` ... LL | check_copy(&gen_non_clone); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}`, the trait `Copy` is not implemented for `NonClone` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Copy` is not implemented for `NonClone` | note: captured value does not implement `Copy` - --> $DIR/clone-impl.rs:84:14 + --> $DIR/clone-impl.rs:83:9 | -LL | drop(non_clonable); - | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Copy` +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Copy` note: required by a bound in `check_copy` - --> $DIR/clone-impl.rs:92:18 + --> $DIR/clone-impl.rs:95:18 | LL | fn check_copy(_x: &T) {} | ^^^^ required by this bound in `check_copy` @@ -85,22 +85,22 @@ LL + #[derive(Copy)] LL | struct NonClone; | -error[E0277]: the trait bound `NonClone: Clone` is not satisfied in `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` - --> $DIR/clone-impl.rs:88:5 +error[E0277]: the trait bound `NonClone: Clone` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:91:5 | LL | move || { - | ------- within this `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}` + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` ... LL | check_clone(&gen_non_clone); - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:82:5: 82:12}`, the trait `Clone` is not implemented for `NonClone` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Clone` is not implemented for `NonClone` | note: captured value does not implement `Clone` - --> $DIR/clone-impl.rs:84:14 + --> $DIR/clone-impl.rs:83:9 | -LL | drop(non_clonable); - | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Clone` +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Clone` note: required by a bound in `check_clone` - --> $DIR/clone-impl.rs:93:19 + --> $DIR/clone-impl.rs:96:19 | LL | fn check_clone(_x: &T) {} | ^^^^^ required by this bound in `check_clone` diff --git a/tests/ui/coroutine/clone-impl.relocate.stderr b/tests/ui/coroutine/clone-impl.relocate.stderr new file mode 100644 index 0000000000000..73513cf8df612 --- /dev/null +++ b/tests/ui/coroutine/clone-impl.relocate.stderr @@ -0,0 +1,115 @@ +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` + --> $DIR/clone-impl.rs:51:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}` +... +LL | check_copy(&gen_clone_0); + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:47:5: 47:12}`, the trait `Copy` is not implemented for `Vec` + | +note: captured value does not implement `Copy` + --> $DIR/clone-impl.rs:45:9 + | +LL | let clonable_0: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` + +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` + --> $DIR/clone-impl.rs:63:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}` +... +LL | check_copy(&gen_clone_1); + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:58:5: 58:12}`, the trait `Copy` is not implemented for `Vec` + | +note: coroutine does not implement `Copy` as this value is used across a yield + --> $DIR/clone-impl.rs:60:9 + | +LL | let v = vec!['a']; + | - has type `Vec` which does not implement `Copy` +LL | yield; + | ^^^^^ yield occurs here, with `v` maybe used later +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` + +error[E0277]: the trait bound `Vec: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` + --> $DIR/clone-impl.rs:77:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}` +... +LL | check_copy(&gen_clone_1); + | ^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:71:5: 71:12}`, the trait `Copy` is not implemented for `Vec` + | +note: captured value does not implement `Copy` + --> $DIR/clone-impl.rs:69:9 + | +LL | let clonable_1: Vec = Vec::new(); + | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` + +error[E0277]: the trait bound `NonClone: Copy` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:89:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` +... +LL | check_copy(&gen_non_clone); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Copy` is not implemented for `NonClone` + | +note: captured value does not implement `Copy` + --> $DIR/clone-impl.rs:83:9 + | +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Copy` +note: required by a bound in `check_copy` + --> $DIR/clone-impl.rs:95:18 + | +LL | fn check_copy(_x: &T) {} + | ^^^^ required by this bound in `check_copy` +help: consider annotating `NonClone` with `#[derive(Copy)]` + | +LL + #[derive(Copy)] +LL | struct NonClone; + | + +error[E0277]: the trait bound `NonClone: Clone` is not satisfied in `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` + --> $DIR/clone-impl.rs:91:5 + | +LL | move || { + | ------- within this `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}` +... +LL | check_clone(&gen_non_clone); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ within `{coroutine@$DIR/clone-impl.rs:85:5: 85:12}`, the trait `Clone` is not implemented for `NonClone` + | +note: captured value does not implement `Clone` + --> $DIR/clone-impl.rs:83:9 + | +LL | let non_clonable: NonClone = NonClone; + | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Clone` +note: required by a bound in `check_clone` + --> $DIR/clone-impl.rs:96:19 + | +LL | fn check_clone(_x: &T) {} + | ^^^^^ required by this bound in `check_clone` +help: consider annotating `NonClone` with `#[derive(Clone)]` + | +LL + #[derive(Clone)] +LL | struct NonClone; + | + +error: aborting due to 5 previous errors + +For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/coroutine/clone-impl.rs b/tests/ui/coroutine/clone-impl.rs index 9e04e256fc11e..17723b56449b3 100644 --- a/tests/ui/coroutine/clone-impl.rs +++ b/tests/ui/coroutine/clone-impl.rs @@ -1,7 +1,10 @@ // gate-test-coroutine_clone // Verifies that non-static coroutines can be cloned/copied if all their upvars and locals held // across awaits can be cloned/copied. -//@compile-flags: --diagnostic-width=300 +//@ revisions: classic relocate +//@ compile-flags: --diagnostic-width=300 +//@ [classic] compile-flags: -Zpack-coroutine-layout=no +//@ [relocate] compile-flags: -Zpack-coroutine-layout=captures-only #![feature(coroutines, coroutine_clone, stmt_expr_attributes)] diff --git a/tests/ui/coroutine/delayed-obligations-emit.next.stderr b/tests/ui/coroutine/delayed-obligations-emit.next.stderr index 3a3663398c9a7..af7dfc75963c2 100644 --- a/tests/ui/coroutine/delayed-obligations-emit.next.stderr +++ b/tests/ui/coroutine/delayed-obligations-emit.next.stderr @@ -4,6 +4,7 @@ error[E0275]: overflow evaluating the requirement `{async block@$DIR/delayed-obl LL | spawn(async { build_dependencies().await }); | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | + = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`delayed_obligations_emit`) note: required by a bound in `spawn` --> $DIR/delayed-obligations-emit.rs:31:13 | diff --git a/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs b/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs index 0f9c56786da06..c862ba658aa01 100644 --- a/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs +++ b/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs @@ -1,16 +1,13 @@ #![feature(coroutines, stmt_expr_attributes)] fn foo(x: &i32) { - // In this case, a reference to `b` escapes the coroutine, but not - // because of a yield. We see that there is no yield in the scope of - // `b` and give the more generic error message. let mut a = &3; - let mut b = #[coroutine] + let b = #[coroutine] move || { yield (); let b = 5; a = &b; - //~^ ERROR borrowed data escapes outside of coroutine + //~^ ERROR: borrowed data escapes outside of coroutine }; } diff --git a/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr b/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr index 6fa7082c0b8b7..41b118d9f8f3a 100644 --- a/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr +++ b/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr @@ -1,5 +1,5 @@ error[E0521]: borrowed data escapes outside of coroutine - --> $DIR/ref-escapes-but-not-over-yield.rs:12:9 + --> $DIR/ref-escapes-but-not-over-yield.rs:9:9 | LL | let mut a = &3; | ----- `a` declared here, outside of the coroutine body diff --git a/tests/ui/coroutine/ref-upvar-not-send.rs b/tests/ui/coroutine/ref-upvar-not-send.rs index 89bb5e5495f45..a843e9ba29a64 100644 --- a/tests/ui/coroutine/ref-upvar-not-send.rs +++ b/tests/ui/coroutine/ref-upvar-not-send.rs @@ -11,21 +11,27 @@ fn assert_send(_: T) {} fn main() { let x: &*mut () = &std::ptr::null_mut(); + //~^ NOTE has type `&*mut ()` which is not `Send` + //~| NOTE captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` let y: &mut *mut () = &mut std::ptr::null_mut(); - assert_send(#[coroutine] move || { + //~^ NOTE has type `&mut *mut ()` which is not `Send` + //~| NOTE captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` + assert_send( //~^ ERROR coroutine cannot be sent between threads safely //~| NOTE coroutine is not `Send` - yield; - let _x = x; - }); - //~^^ NOTE captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - //~| NOTE has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` - assert_send(#[coroutine] move || { + #[coroutine] + move || { + yield; + let _x = x; + }, + ); + assert_send( //~^ ERROR coroutine cannot be sent between threads safely //~| NOTE coroutine is not `Send` - yield; - let _y = y; - }); - //~^^ NOTE captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` - //~| NOTE has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` + #[coroutine] + move || { + yield; + let _y = y; + }, + ); } diff --git a/tests/ui/coroutine/ref-upvar-not-send.stderr b/tests/ui/coroutine/ref-upvar-not-send.stderr index 3a5e8ec4dab0c..1197583c0cc08 100644 --- a/tests/ui/coroutine/ref-upvar-not-send.stderr +++ b/tests/ui/coroutine/ref-upvar-not-send.stderr @@ -1,20 +1,21 @@ error: coroutine cannot be sent between threads safely - --> $DIR/ref-upvar-not-send.rs:15:5 + --> $DIR/ref-upvar-not-send.rs:19:5 | -LL | / assert_send(#[coroutine] move || { +LL | / assert_send( LL | | LL | | -LL | | yield; -LL | | let _x = x; -LL | | }); - | |______^ coroutine is not `Send` +LL | | #[coroutine] +... | +LL | | }, +LL | | ); + | |_____^ coroutine is not `Send` | = help: the trait `Sync` is not implemented for `*mut ()` note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> $DIR/ref-upvar-not-send.rs:19:18 + --> $DIR/ref-upvar-not-send.rs:13:9 | -LL | let _x = x; - | ^ has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` +LL | let x: &*mut () = &std::ptr::null_mut(); + | ^ has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` note: required by a bound in `assert_send` --> $DIR/ref-upvar-not-send.rs:6:19 | @@ -22,22 +23,23 @@ LL | fn assert_send(_: T) {} | ^^^^ required by this bound in `assert_send` error: coroutine cannot be sent between threads safely - --> $DIR/ref-upvar-not-send.rs:23:5 + --> $DIR/ref-upvar-not-send.rs:28:5 | -LL | / assert_send(#[coroutine] move || { +LL | / assert_send( LL | | LL | | -LL | | yield; -LL | | let _y = y; -LL | | }); - | |______^ coroutine is not `Send` +LL | | #[coroutine] +... | +LL | | }, +LL | | ); + | |_____^ coroutine is not `Send` | - = help: within `{coroutine@$DIR/ref-upvar-not-send.rs:23:30: 23:37}`, the trait `Send` is not implemented for `*mut ()` + = help: within `{coroutine@$DIR/ref-upvar-not-send.rs:32:9: 32:16}`, the trait `Send` is not implemented for `*mut ()` note: captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` - --> $DIR/ref-upvar-not-send.rs:27:18 + --> $DIR/ref-upvar-not-send.rs:16:9 | -LL | let _y = y; - | ^ has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` +LL | let y: &mut *mut () = &mut std::ptr::null_mut(); + | ^ has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` note: required by a bound in `assert_send` --> $DIR/ref-upvar-not-send.rs:6:19 | diff --git a/tests/ui/coroutine/unsized-capture-across-yield.rs b/tests/ui/coroutine/unsized-capture-across-yield.rs index ee27ea064ec23..60c8f69bdb8f0 100644 --- a/tests/ui/coroutine/unsized-capture-across-yield.rs +++ b/tests/ui/coroutine/unsized-capture-across-yield.rs @@ -4,7 +4,8 @@ use std::ops::Coroutine; fn capture() -> impl Coroutine { - let b: [u8] = *(Box::new([]) as Box<[u8]>); //~ERROR he size for values of type `[u8]` cannot be known at compilation time + let b: [u8] = *(Box::new([]) as Box<[u8]>); + //~^ ERROR the size for values of type `[u8]` cannot be known at compilation time #[coroutine] move || { println!("{:?}", &b); diff --git a/tests/ui/lint/large_assignments/large_future.attribute.stderr b/tests/ui/lint/large_assignments/large_future.attribute.stderr index 734b7ff7ba22f..3c3a1fb3e5fb8 100644 --- a/tests/ui/lint/large_assignments/large_future.attribute.stderr +++ b/tests/ui/lint/large_assignments/large_future.attribute.stderr @@ -1,5 +1,5 @@ error: moving 10024 bytes - --> $DIR/large_future.rs:19:14 + --> $DIR/large_future.rs:21:14 | LL | let z = (x, 42); | ^ value moved from here @@ -12,7 +12,7 @@ LL | #![deny(large_assignments)] | ^^^^^^^^^^^^^^^^^ error: moving 10024 bytes - --> $DIR/large_future.rs:20:13 + --> $DIR/large_future.rs:22:13 | LL | let a = z.0; | ^^^ value moved from here diff --git a/tests/ui/lint/large_assignments/large_future.option-relocate.stderr b/tests/ui/lint/large_assignments/large_future.option-relocate.stderr new file mode 100644 index 0000000000000..3c3a1fb3e5fb8 --- /dev/null +++ b/tests/ui/lint/large_assignments/large_future.option-relocate.stderr @@ -0,0 +1,23 @@ +error: moving 10024 bytes + --> $DIR/large_future.rs:21:14 + | +LL | let z = (x, 42); + | ^ value moved from here + | + = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]` +note: the lint level is defined here + --> $DIR/large_future.rs:1:9 + | +LL | #![deny(large_assignments)] + | ^^^^^^^^^^^^^^^^^ + +error: moving 10024 bytes + --> $DIR/large_future.rs:22:13 + | +LL | let a = z.0; + | ^^^ value moved from here + | + = note: The current maximum size is 1000, but it can be customized with the move_size_limit attribute: `#![move_size_limit = "..."]` + +error: aborting due to 2 previous errors + diff --git a/tests/ui/lint/large_assignments/large_future.option.stderr b/tests/ui/lint/large_assignments/large_future.option.stderr index 734b7ff7ba22f..3c3a1fb3e5fb8 100644 --- a/tests/ui/lint/large_assignments/large_future.option.stderr +++ b/tests/ui/lint/large_assignments/large_future.option.stderr @@ -1,5 +1,5 @@ error: moving 10024 bytes - --> $DIR/large_future.rs:19:14 + --> $DIR/large_future.rs:21:14 | LL | let z = (x, 42); | ^ value moved from here @@ -12,7 +12,7 @@ LL | #![deny(large_assignments)] | ^^^^^^^^^^^^^^^^^ error: moving 10024 bytes - --> $DIR/large_future.rs:20:13 + --> $DIR/large_future.rs:22:13 | LL | let a = z.0; | ^^^ value moved from here diff --git a/tests/ui/lint/large_assignments/large_future.rs b/tests/ui/lint/large_assignments/large_future.rs index 28c358bdbf086..43ae254663be9 100644 --- a/tests/ui/lint/large_assignments/large_future.rs +++ b/tests/ui/lint/large_assignments/large_future.rs @@ -3,8 +3,10 @@ #![cfg_attr(attribute, move_size_limit = "1000")] //@ build-fail //@ only-64bit -//@ revisions: attribute option -//@ [option]compile-flags: -Zmove-size-limit=1000 +//@ revisions: attribute option option-relocate +//@ compile-flags: -Zpack-coroutine-layout=no +//@ [option] compile-flags: -Zmove-size-limit=1000 +//@ [option-relocate] compile-flags: -Zmove-size-limit=1000 -Zpack-coroutine-layout=captures-only //@ edition:2018 //@ compile-flags: -Zmir-opt-level=0 diff --git a/tests/ui/mir/lint/storage-live.stderr b/tests/ui/mir/lint/storage-live.stderr index 50df9ae061fcc..fabf7f33745f0 100644 --- a/tests/ui/mir/lint/storage-live.stderr +++ b/tests/ui/mir/lint/storage-live.stderr @@ -1,4 +1,4 @@ -error: internal compiler error: broken MIR in Item(DefId(0:8 ~ storage_live[HASH]::multiple_storage)) (after pass CheckForceInline) at bb0[1]: +error: internal compiler error: broken MIR in Item(DefId(0:8 ~ storage_live[HASH]::multiple_storage)) (after pass CheckForceInline 1-1-000) at bb0[1]: StorageLive(_1) which already has storage here --> $DIR/storage-live.rs:21:13 | diff --git a/tests/ui/print_type_sizes/async.stdout b/tests/ui/print_type_sizes/async.classic.stdout similarity index 92% rename from tests/ui/print_type_sizes/async.stdout rename to tests/ui/print_type_sizes/async.classic.stdout index d3d6b6471c6ef..dcc0585a69576 100644 --- a/tests/ui/print_type_sizes/async.stdout +++ b/tests/ui/print_type_sizes/async.classic.stdout @@ -1,15 +1,14 @@ print-type-size type: `{async fn body of test()}`: 16386 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes +print-type-size local `.arg`: 8192 bytes +print-type-size upvar `.arg`: 8192 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Suspend0`: 16385 bytes -print-type-size upvar `.arg`: 8192 bytes +print-type-size padding: 8192 bytes +print-type-size local `.__awaitee`: 1 bytes, alignment: 1 bytes, type: {async fn body of wait()} print-type-size local `.arg`: 8192 bytes -print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} print-type-size variant `Returned`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes print-type-size variant `Panicked`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes print-type-size field `.value`: 8192 bytes print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes diff --git a/tests/ui/print_type_sizes/async.relocate.stdout b/tests/ui/print_type_sizes/async.relocate.stdout new file mode 100644 index 0000000000000..c6a7d8723e5c8 --- /dev/null +++ b/tests/ui/print_type_sizes/async.relocate.stdout @@ -0,0 +1,65 @@ +print-type-size type: `{async fn body of test()}`: 8194 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 8192 bytes +print-type-size upvar `.arg`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.arg`: 8192 bytes +print-type-size variant `Suspend0`: 8193 bytes +print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} +print-type-size local `.arg`: 8192 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::task::Context<'_>`: 32 bytes, alignment: 8 bytes +print-type-size field `.waker`: 8 bytes +print-type-size field `.local_waker`: 8 bytes +print-type-size field `.ext`: 16 bytes +print-type-size field `._marker`: 0 bytes +print-type-size field `._marker2`: 0 bytes +print-type-size type: `std::panic::Location<'_>`: 24 bytes, alignment: 8 bytes +print-type-size field `.filename`: 16 bytes +print-type-size field `.line`: 4 bytes +print-type-size field `.col`: 4 bytes +print-type-size field `._filename`: 0 bytes +print-type-size type: `core::task::wake::ExtData<'_>`: 16 bytes, alignment: 8 bytes +print-type-size variant `Some`: 16 bytes +print-type-size field `.0`: 16 bytes +print-type-size variant `None`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size type: `std::panic::AssertUnwindSafe>`: 16 bytes, alignment: 8 bytes +print-type-size field `.0`: 16 bytes +print-type-size type: `std::ptr::NonNull`: 16 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 16 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of test()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::pin::Pin<&mut {async fn body of wait()}>`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::ptr::DynMetadata`: 8 bytes, alignment: 8 bytes +print-type-size field `._vtable_ptr`: 8 bytes +print-type-size field `._phantom`: 0 bytes +print-type-size type: `std::ptr::NonNull`: 8 bytes, alignment: 8 bytes +print-type-size field `.pointer`: 8 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1 bytes +print-type-size type: `std::task::Poll<()>`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Ready`: 0 bytes +print-type-size field `.0`: 0 bytes +print-type-size variant `Pending`: 0 bytes +print-type-size type: `{async fn body of wait()}`: 1 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 0 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::marker::PhantomData<&str>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData<*mut ()>`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData`: 0 bytes, alignment: 1 bytes +print-type-size type: `std::marker::PhantomData &()>`: 0 bytes, alignment: 1 bytes diff --git a/tests/ui/print_type_sizes/async.rs b/tests/ui/print_type_sizes/async.rs index b6ec88426345b..4020586b64f04 100644 --- a/tests/ui/print_type_sizes/async.rs +++ b/tests/ui/print_type_sizes/async.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only // FIXME(#61117): Respect debuginfo-level-tests, do not force debuginfo=0 //@ compile-flags: -C debuginfo=0 //@ compile-flags: -C panic=abort -Z print-type-sizes --crate-type lib diff --git a/tests/ui/print_type_sizes/coroutine.classic.stdout b/tests/ui/print_type_sizes/coroutine.classic.stdout new file mode 100644 index 0000000000000..d4e9019bb6cec --- /dev/null +++ b/tests/ui/print_type_sizes/coroutine.classic.stdout @@ -0,0 +1,14 @@ +print-type-size type: `{coroutine@$DIR/coroutine.rs:14:5: 14:14}`: 8193 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 8192 bytes +print-type-size local `.array`: 8192 bytes +print-type-size upvar `.array`: 8192 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 8192 bytes +print-type-size variant `Returned`: 8192 bytes +print-type-size variant `Panicked`: 8192 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes diff --git a/tests/ui/print_type_sizes/coroutine.relocate.stdout b/tests/ui/print_type_sizes/coroutine.relocate.stdout new file mode 100644 index 0000000000000..79f5d093dc3ad --- /dev/null +++ b/tests/ui/print_type_sizes/coroutine.relocate.stdout @@ -0,0 +1,15 @@ +print-type-size type: `{coroutine@$DIR/coroutine.rs:14:5: 14:14}`: 8193 bytes, alignment: 1 bytes +print-type-size discriminant: 1 bytes +print-type-size variant `Unresumed`: 8192 bytes +print-type-size upvar `.array`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.array`: 8192 bytes +print-type-size variant `Suspend0`: 8192 bytes +print-type-size local `.array`: 8192 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes diff --git a/tests/ui/print_type_sizes/coroutine.rs b/tests/ui/print_type_sizes/coroutine.rs index 1533578878944..407d0873c015d 100644 --- a/tests/ui/print_type_sizes/coroutine.rs +++ b/tests/ui/print_type_sizes/coroutine.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ compile-flags: -Z print-type-sizes --crate-type=lib //@ build-pass //@ ignore-pass diff --git a/tests/ui/print_type_sizes/coroutine.stdout b/tests/ui/print_type_sizes/coroutine.stdout deleted file mode 100644 index 339bbddfc2a93..0000000000000 --- a/tests/ui/print_type_sizes/coroutine.stdout +++ /dev/null @@ -1,10 +0,0 @@ -print-type-size type: `{coroutine@$DIR/coroutine.rs:11:5: 11:14}`: 8193 bytes, alignment: 1 bytes -print-type-size discriminant: 1 bytes -print-type-size variant `Unresumed`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Suspend0`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Returned`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Panicked`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes diff --git a/tests/ui/rustc_public-ir-print/async-closure.stdout b/tests/ui/rustc_public-ir-print/async-closure.classic.stdout similarity index 70% rename from tests/ui/rustc_public-ir-print/async-closure.stdout rename to tests/ui/rustc_public-ir-print/async-closure.classic.stdout index 3ec816b657f45..ad702cd1cee66 100644 --- a/tests/ui/rustc_public-ir-print/async-closure.stdout +++ b/tests/ui/rustc_public-ir-print/async-closure.classic.stdout @@ -3,7 +3,7 @@ fn foo() -> () { let mut _0: (); let _1: i32; - let _2: {async closure@$DIR/async-closure.rs:9:13: 9:21}; + let _2: {async closure@$DIR/async-closure.rs:12:13: 12:21}; let mut _3: &i32; debug y => _1; debug x => _2; @@ -13,7 +13,7 @@ fn foo() -> () { StorageLive(_2); StorageLive(_3); _3 = &_1; - _2 = {coroutine-closure@$DIR/async-closure.rs:9:13: 9:21}(move _3); + _2 = {coroutine-closure@$DIR/async-closure.rs:12:13: 12:21}(move _3); StorageDead(_3); _0 = (); StorageDead(_2); @@ -21,8 +21,8 @@ fn foo() -> () { return; } } -fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:9:13: 9:21}) -> {async closure body@$DIR/async-closure.rs:9:22: 11:6} { - let mut _0: {async closure body@$DIR/async-closure.rs:9:22: 11:6}; +fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:12:13: 12:21}) -> {async closure body@$DIR/async-closure.rs:12:22: 14:6} { + let mut _0: {async closure body@$DIR/async-closure.rs:12:22: 14:6}; let mut _2: &i32; let mut _3: &i32; debug y => (*((*_1).0: &i32)); @@ -30,29 +30,29 @@ fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:9:13: 9:21}) -> {a StorageLive(_2); _3 = ((*_1).0: &i32); _2 = &(*_3); - _0 = {coroutine@$DIR/async-closure.rs:9:22: 11:6}(move _2); + _0 = {coroutine@$DIR/async-closure.rs:12:22: 14:6}(move _2); StorageDead(_2); return; } } -fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}>, _2: &mut Context<'_>) -> Poll<()> { +fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { let mut _0: Poll<()>; let _3: i32; let mut _4: &i32; let mut _5: (); let mut _6: u32; - let mut _7: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; + let mut _7: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; debug _task_context => _2; - debug y => (*((*_7).0: &i32)); + debug y => (*(((*_7) as variant#0).0: &i32)); debug y => _3; bb0: { - _7 = (_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}); + _7 = (_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}); _6 = discriminant((*_7)); switchInt(move _6) -> [0: bb1, 1: bb2, otherwise: bb3]; } bb1: { StorageLive(_3); - _4 = ((*_7).0: &i32); + _4 = (((*_7) as variant#0).0: &i32); _3 = (*_4); _5 = (); StorageDead(_3); @@ -67,24 +67,24 @@ fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-clo unreachable; } } -fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}>, _2: &mut Context<'_>) -> Poll<()> { +fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { let mut _0: Poll<()>; let _3: i32; let mut _4: &i32; let mut _5: (); let mut _6: u32; - let mut _7: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}; + let mut _7: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; debug _task_context => _2; - debug y => (*((*_7).0: &i32)); + debug y => (*(((*_7) as variant#0).0: &i32)); debug y => _3; bb0: { - _7 = (_1.0: &mut {async closure body@$DIR/async-closure.rs:9:22: 11:6}); + _7 = (_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}); _6 = discriminant((*_7)); switchInt(move _6) -> [0: bb1, 1: bb2, otherwise: bb3]; } bb1: { StorageLive(_3); - _4 = ((*_7).0: &i32); + _4 = (((*_7) as variant#0).0: &i32); _3 = (*_4); _5 = (); StorageDead(_3); diff --git a/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout b/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout new file mode 100644 index 0000000000000..d1fe068b9c253 --- /dev/null +++ b/tests/ui/rustc_public-ir-print/async-closure.relocate.stdout @@ -0,0 +1,119 @@ +// WARNING: This is highly experimental output it's intended for rustc_public developers only. +// If you find a bug or want to improve the output open a issue at https://github.com/rust-lang/project-stable-mir. +fn foo() -> () { + let mut _0: (); + let _1: i32; + let _2: {async closure@$DIR/async-closure.rs:12:13: 12:21}; + let mut _3: &i32; + debug y => _1; + debug x => _2; + bb0: { + StorageLive(_1); + _1 = 0_i32; + StorageLive(_2); + StorageLive(_3); + _3 = &_1; + _2 = {coroutine-closure@$DIR/async-closure.rs:12:13: 12:21}(move _3); + StorageDead(_3); + _0 = (); + StorageDead(_2); + StorageDead(_1); + return; + } +} +fn foo::{closure#0}(_1: &{async closure@$DIR/async-closure.rs:12:13: 12:21}) -> {async closure body@$DIR/async-closure.rs:12:22: 14:6} { + let mut _0: {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + let mut _2: &i32; + let mut _3: &i32; + debug y => (*((*_1).0: &i32)); + bb0: { + StorageLive(_2); + _3 = ((*_1).0: &i32); + _2 = &(*_3); + _0 = {coroutine@$DIR/async-closure.rs:12:22: 14:6}(move _2); + StorageDead(_2); + return; + } +} +fn foo::{closure#0}::{closure#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { + let mut _0: Poll<()>; + let _3: i32; + let mut _4: &i32; + let mut _5: &i32; + let mut _6: &i32; + let mut _7: (); + let mut _8: u32; + let mut _9: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + debug _task_context => _2; + debug y => (*_5); + debug y => _3; + debug y => _5; + bb0: { + _9 = (_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}); + _8 = discriminant((*_9)); + switchInt(move _8) -> [0: bb1, 1: bb2, otherwise: bb3]; + } + bb1: { + StorageLive(_5); + StorageLive(_6); + _6 = move (((*_9) as variant#0).0: &i32); + _5 = move _6; + StorageDead(_6); + StorageLive(_3); + _4 = _5; + _3 = (*_4); + _7 = (); + StorageDead(_3); + StorageDead(_5); + _0 = std::task::Poll::Ready(move _7); + discriminant((*_9)) = 1; + return; + } + bb2: { + assert(false, `async fn` resumed after completion) -> [success: bb2, unwind unreachable]; + } + bb3: { + unreachable; + } +} +fn foo::{closure#0}::{synthetic#0}(_1: Pin<&mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}>, _2: &mut Context<'_>) -> Poll<()> { + let mut _0: Poll<()>; + let _3: i32; + let mut _4: &i32; + let mut _5: &i32; + let mut _6: &i32; + let mut _7: (); + let mut _8: u32; + let mut _9: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}; + debug _task_context => _2; + debug y => (*_5); + debug y => _3; + debug y => _5; + bb0: { + _9 = (_1.0: &mut {async closure body@$DIR/async-closure.rs:12:22: 14:6}); + _8 = discriminant((*_9)); + switchInt(move _8) -> [0: bb1, 1: bb2, otherwise: bb3]; + } + bb1: { + StorageLive(_5); + StorageLive(_6); + _6 = move (((*_9) as variant#0).0: &i32); + _5 = move _6; + StorageDead(_6); + StorageLive(_3); + _4 = _5; + _3 = (*_4); + _7 = (); + StorageDead(_3); + StorageDead(_5); + _0 = std::task::Poll::Ready(move _7); + discriminant((*_9)) = 1; + return; + } + bb2: { + assert(false, `async fn` resumed after completion) -> [success: bb2, unwind unreachable]; + } + bb3: { + unreachable; + } +} diff --git a/tests/ui/rustc_public-ir-print/async-closure.rs b/tests/ui/rustc_public-ir-print/async-closure.rs index bd8c7e888a37c..4457468752b2e 100644 --- a/tests/ui/rustc_public-ir-print/async-closure.rs +++ b/tests/ui/rustc_public-ir-print/async-closure.rs @@ -1,3 +1,6 @@ +//@ revisions: classic relocate +//@ [classic] compile-flags: -Z pack-coroutine-layout=no +//@ [relocate] compile-flags: -Z pack-coroutine-layout=captures-only //@ compile-flags: -Z unpretty=stable-mir --crate-type lib -C panic=abort -Zmir-opt-level=0 //@ check-pass //@ only-64bit