-Znext-solver Prevent committing unfulfilled unsized coercion

compiler/rustc_hir_typeck/src/coercion.rs

@@ -644,7 +644,7 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
                 .infcx
                 .visit_proof_tree(
                     Goal::new(self.tcx, self.param_env, pred),
-                    &mut CoerceVisitor { fcx: self.fcx, span: self.cause.span },
+                    &mut CoerceVisitor { fcx: self.fcx, span: self.cause.span, errored: false },
                 )
                 .is_break()
             {
@@ -1961,6 +1961,10 @@ impl<'tcx> CoerceMany<'tcx> {
 struct CoerceVisitor<'a, 'tcx> {
     fcx: &'a FnCtxt<'a, 'tcx>,
     span: Span,
+    /// Whether the coercion is impossible. If so we sometimes still try to
+    /// coerce in these cases to emit better errors. This changes the behavior
+    /// when hitting the recursion limit.
+    errored: bool,
 }
 
 impl<'tcx> ProofTreeVisitor<'tcx> for CoerceVisitor<'_, 'tcx> {
@@ -1987,6 +1991,7 @@ impl<'tcx> ProofTreeVisitor<'tcx> for CoerceVisitor<'_, 'tcx> {
             // If we prove the `Unsize` or `CoerceUnsized` goal, continue recursing.
             Ok(Certainty::Yes) => ControlFlow::Continue(()),
             Err(NoSolution) => {
+                self.errored = true;
                 // Even if we find no solution, continue recursing if we find a single candidate
                 // for which we're shallowly certain it holds to get the right error source.
                 if let [only_candidate] = &goal.candidates()[..]
@@ -2019,4 +2024,15 @@ impl<'tcx> ProofTreeVisitor<'tcx> for CoerceVisitor<'_, 'tcx> {
             }
         }
     }
+
+    fn on_recursion_limit(&mut self) -> Self::Result {
+        if self.errored {
+            // This prevents accidentally committing unfulfilled unsized coercions while trying to
+            // find the error source for diagnostics.
+            // See https://github.com/rust-lang/trait-system-refactor-initiative/issues/266.
+            ControlFlow::Break(())
+        } else {
+            ControlFlow::Continue(())
+        }
+    }
 }

compiler/rustc_trait_selection/src/solve/inspect/analyse.rs

@@ -443,9 +443,10 @@ impl<'a, 'tcx> InspectGoal<'a, 'tcx> {
     pub(crate) fn visit_with<V: ProofTreeVisitor<'tcx>>(&self, visitor: &mut V) -> V::Result {
         if self.depth < visitor.config().max_depth {
             try_visit!(visitor.visit_goal(self));
+            V::Result::output()
+        } else {
+            visitor.on_recursion_limit()
         }
-
-        V::Result::output()
     }
 }
 
@@ -460,6 +461,10 @@ pub trait ProofTreeVisitor<'tcx> {
     }
 
     fn visit_goal(&mut self, goal: &InspectGoal<'_, 'tcx>) -> Self::Result;
+
+    fn on_recursion_limit(&mut self) -> Self::Result {
+        Self::Result::output()
+    }
 }
 
 #[extension(pub trait InferCtxtProofTreeExt<'tcx>)]

tests/ui/traits/next-solver/coercion/unfulfilled-unsize-coercion-recursion-limit.rs

@@ -0,0 +1,35 @@
+//@ check-pass
+//@ compile-flags: -Znext-solver
+
+// A regression test for https://github.com/rust-lang/trait-system-refactor-initiative/issues/266.
+// Ensure that we do not accidentaly trying unfulfilled unsized coercions due to hitting recursion
+// limits while trying to find the right fulfillment error source.
+
+fn argument_coercion<U>(_: &U) {}
+
+pub fn test() {
+    argument_coercion(&{
+        Nested(0.0, 0.0)
+            .add(0.0)
+            .add(0.0)
+            .add(0.0)
+            .add(0.0)
+            .add(0.0)
+            .add(0.0)
+            .add(0.0)
+            .add(0.0)
+            .add(0.0)
+            .add(0.0)
+            .add(0.0)
+    });
+}
+
+struct Nested<T, R>(T, R);
+
+impl<T, R> Nested<T, R> {
+    fn add<U>(self, value: U) -> Nested<U, Nested<T, R>> {
+        Nested(value, self)
+    }
+}
+
+fn main() {}

tests/ui/traits/next-solver/coercion/unsize-coercion-recursion-limit.rs

@@ -0,0 +1,25 @@
+//@ check-pass
+//@ compile-flags: -Znext-solver
+
+// A test to ensure that unsized coercion is not aborted when visiting a nested goal that
+// exceeds the recursion limit and evaluates to `Certainty::Maybe`.
+// See https://github.com/rust-lang/rust/pull/152444.
+
+#![allow(warnings)]
+
+struct W<T: ?Sized>(T);
+type Four<T: ?Sized> = W<W<W<W<T>>>>;
+type Sixteen<T: ?Sized> = Four<Four<Four<Four<T>>>>;
+
+fn ret<T>(x: T) -> Sixteen<T> {
+    todo!();
+}
+
+fn please_coerce() {
+    let mut y = Default::default();
+    let x = ret(y);
+    let _: &Sixteen<dyn Send> = &x;
+    y = 1u32;
+}
+
+fn main() {}