[Fuzzlyn] Assert in `isEmbeddedMaskingCompatible`

[tannergooding]

The following code asserts:

// Generated by Fuzzlyn v3.3 on 2025-07-14 11:26:52
// Run on X64 Windows
// Seed: 5654087083843205658-vectort,vector128,vector256,x86aes,x86avx,x86avx2,x86avx512bw,x86avx512bwvl,x86avx512cd,x86avx512cdvl,x86avx512dq,x86avx512dqvl,x86avx512f,x86avx512fvl,x86avx512fx64,x86bmi1,x86bmi1x64,x86bmi2,x86bmi2x64,x86fma,x86lzcnt,x86lzcntx64,x86pclmulqdq,x86popcnt,x86popcntx64,x86sse,x86ssex64,x86sse2,x86sse2x64,x86sse3,x86sse41,x86sse41x64,x86sse42,x86sse42x64,x86ssse3,x86x86base
// Reduced from 33.4 KiB to 0.9 KiB in 00:01:30
// Hits JIT assert for Release:
// Assertion failed '(maskBaseSize == 4) || (maskBaseSize == 8)' in 'Program:M0()' during 'Rationalize IR' (IL size 87; hash 0xaf50ff37; FullOpts)
// 
//     File: D:\a\_work\1\s\src\coreclr\jit\gentree.cpp Line: 20819
// 
using System.Numerics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;

public class C0
{
    public short M3(ref Vector<uint>[] arg0, Vector256<ulong> arg1)
    {
        var vr5 = Vector128.Create(0, -1, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0);
        return (short)Sse2.MoveMask(vr5);
    }
}

public class Program
{
    public static Vector<uint>[] s_2;
    public static void Main()
    {
        M0();
    }

    public static void M0()
    {
        var vr3 = Vector256.Create(0, 7424648407429701945UL, 0, 0);
        var vr6 = Vector256.Create<ulong>(0);
        var vr7 = Vector128.CreateScalar(9831122154695836571UL);
        var vr4 = Avx2.InsertVector128(vr6, vr7, 0);
        var vr8 = Vector256.CreateScalar(1497050855019840058UL);
        var vr2 = Avx2.BlendVariable(vr3, vr4, vr8);
        C0 vr9 = default(C0);
        var vr1 = vr9.M3(ref s_2, vr2);
    }
}

The issue is it should be checking (maskBaseSize == 2) || (maskBaseSize == 4) (rather than checking (maskBaseSize == 8)) because Vector128<float> contains 4x float (size 4) or 2x double (size 8).

[dotnet-policy-service]

Tagging subscribers to this area: @JulieLeeMSFT, @jakobbotsch
See info in area-owners.md if you want to be subscribed.