Replace assertion in limit_type_size with fall-back (#36516)

While the condition (the new type has to be wider) of the assertion
should hold, #36407 shows that this may still fail sometimes. Instead of
throwing an error, it seems better to just widen more aggressively if
needed to ensure that the condition is fulfilled.

(cherry picked from commit 604f658)

base/compiler/typelimits.jl

@@ -20,7 +20,13 @@ function limit_type_size(@nospecialize(t), @nospecialize(compare), @nospecialize
     source[1] === source[2] && (source = svec(source[1]))
     type_more_complex(t, compare, source, 1, allowed_tupledepth, allowed_tuplelen) || return t
     r = _limit_type_size(t, compare, source, 1, allowed_tuplelen)
-    @assert t <: r
+    #@assert t <: r # this may fail if t contains a typevar in invariant and multiple times
+        # in covariant position and r looses the occurence in invariant position (see #36407)
+    if !(t <: r) # ideally, this should never happen
+        # widen to minimum complexity to obtain a valid result
+        r = _limit_type_size(t, Any, source, 1, allowed_tuplelen)
+        t <: r || (r = Any) # final escape hatch
+    end
     #@assert r === _limit_type_size(r, t, source) # this monotonicity constraint is slightly stronger than actually required,
       # since we only actually need to demonstrate that repeated application would reaches a fixed point,
       #not that it is already at the fixed point

test/compiler/inference.jl

@@ -32,6 +32,9 @@ let ref = Tuple{T, Val{T}} where T<:(Val{T} where T<:(Val{T} where T<:(Val{T} wh
     @test Core.Compiler.limit_type_size(ref, sig, Union{}, 100, 100) == ref
 end
 
+let t = Tuple{Ref{T},T,T} where T, c = Tuple{Ref, T, T} where T # #36407
+    @test t <: Core.Compiler.limit_type_size(t, c, Union{}, 1, 100)
+end
 
 @test Core.Compiler.unionlen(Union{}) == 1
 @test Core.Compiler.unionlen(Int8) == 1