Add CPUID for AvxVnniInt8 and AvxVnniInt16

src/coreclr/inc/clrconfigvalues.h

@@ -684,6 +684,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAES,                    W("EnableAES"),
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVX512VP2INTERSECT,     W("EnableAVX512VP2INTERSECT"),  1, "Allows AVX512VP2INTERSECT and dependent hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVXIFMA,                W("EnableAVXIFMA"),             1, "Allows AVXIFMA and dependent hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVXVNNI,                W("EnableAVXVNNI"),             1, "Allows AVXVNNI and dependent hardware intrinsics to be disabled")
+RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableAVXVNNIINT,             W("EnableAVXVNNIINT"),          1, "Allows VEX versions (AVXVNNI8 & AVXVNNIINT16) hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableGFNI,                   W("EnableGFNI"),                1, "Allows GFNI and dependent hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableSHA,                    W("EnableSHA"),                 1, "Allows SHA and dependent hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableVAES,                   W("EnableVAES"),                1, "Allows VAES, VPCLMULQDQ, and dependent hardware intrinsics to be disabled")

src/coreclr/inc/corinfoinstructionset.h

@@ -78,23 +78,25 @@ enum CORINFO_InstructionSet
     InstructionSet_VectorT128=26,
     InstructionSet_VectorT256=27,
     InstructionSet_VectorT512=28,
-    InstructionSet_X86Base_X64=29,
-    InstructionSet_SSE42_X64=30,
-    InstructionSet_AVX_X64=31,
-    InstructionSet_AVX2_X64=32,
-    InstructionSet_AVX512_X64=33,
-    InstructionSet_AVX512v2_X64=34,
-    InstructionSet_AVX512v3_X64=35,
-    InstructionSet_AVX10v1_X64=36,
-    InstructionSet_AVX10v2_X64=37,
-    InstructionSet_AES_X64=38,
-    InstructionSet_AVX512VP2INTERSECT_X64=39,
-    InstructionSet_AVXIFMA_X64=40,
-    InstructionSet_AVXVNNI_X64=41,
-    InstructionSet_GFNI_X64=42,
-    InstructionSet_SHA_X64=43,
-    InstructionSet_WAITPKG_X64=44,
-    InstructionSet_X86Serialize_X64=45,
+    InstructionSet_AVXVNNIINT=29,
+    InstructionSet_AVXVNNIINT_V512=30,
+    InstructionSet_X86Base_X64=31,
+    InstructionSet_SSE42_X64=32,
+    InstructionSet_AVX_X64=33,
+    InstructionSet_AVX2_X64=34,
+    InstructionSet_AVX512_X64=35,
+    InstructionSet_AVX512v2_X64=36,
+    InstructionSet_AVX512v3_X64=37,
+    InstructionSet_AVX10v1_X64=38,
+    InstructionSet_AVX10v2_X64=39,
+    InstructionSet_AES_X64=40,
+    InstructionSet_AVX512VP2INTERSECT_X64=41,
+    InstructionSet_AVXIFMA_X64=42,
+    InstructionSet_AVXVNNI_X64=43,
+    InstructionSet_GFNI_X64=44,
+    InstructionSet_SHA_X64=45,
+    InstructionSet_WAITPKG_X64=46,
+    InstructionSet_X86Serialize_X64=47,
 #endif // TARGET_AMD64
 #ifdef TARGET_X86
     InstructionSet_X86Base=1,
@@ -125,23 +127,25 @@ enum CORINFO_InstructionSet
     InstructionSet_VectorT128=26,
     InstructionSet_VectorT256=27,
     InstructionSet_VectorT512=28,
-    InstructionSet_X86Base_X64=29,
-    InstructionSet_SSE42_X64=30,
-    InstructionSet_AVX_X64=31,
-    InstructionSet_AVX2_X64=32,
-    InstructionSet_AVX512_X64=33,
-    InstructionSet_AVX512v2_X64=34,
-    InstructionSet_AVX512v3_X64=35,
-    InstructionSet_AVX10v1_X64=36,
-    InstructionSet_AVX10v2_X64=37,
-    InstructionSet_AES_X64=38,
-    InstructionSet_AVX512VP2INTERSECT_X64=39,
-    InstructionSet_AVXIFMA_X64=40,
-    InstructionSet_AVXVNNI_X64=41,
-    InstructionSet_GFNI_X64=42,
-    InstructionSet_SHA_X64=43,
-    InstructionSet_WAITPKG_X64=44,
-    InstructionSet_X86Serialize_X64=45,
+    InstructionSet_AVXVNNIINT=29,
+    InstructionSet_AVXVNNIINT_V512=30,
+    InstructionSet_X86Base_X64=31,
+    InstructionSet_SSE42_X64=32,
+    InstructionSet_AVX_X64=33,
+    InstructionSet_AVX2_X64=34,
+    InstructionSet_AVX512_X64=35,
+    InstructionSet_AVX512v2_X64=36,
+    InstructionSet_AVX512v3_X64=37,
+    InstructionSet_AVX10v1_X64=38,
+    InstructionSet_AVX10v2_X64=39,
+    InstructionSet_AES_X64=40,
+    InstructionSet_AVX512VP2INTERSECT_X64=41,
+    InstructionSet_AVXIFMA_X64=42,
+    InstructionSet_AVXVNNI_X64=43,
+    InstructionSet_GFNI_X64=44,
+    InstructionSet_SHA_X64=45,
+    InstructionSet_WAITPKG_X64=46,
+    InstructionSet_X86Serialize_X64=47,
 #endif // TARGET_X86
 
 };
@@ -503,6 +507,10 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins
             resultflags.RemoveInstructionSet(InstructionSet_WAITPKG);
         if (resultflags.HasInstructionSet(InstructionSet_X86Serialize) && !resultflags.HasInstructionSet(InstructionSet_X86Base))
             resultflags.RemoveInstructionSet(InstructionSet_X86Serialize);
+        if (resultflags.HasInstructionSet(InstructionSet_AVXVNNIINT) && !resultflags.HasInstructionSet(InstructionSet_AVX2))
+            resultflags.RemoveInstructionSet(InstructionSet_AVXVNNIINT);
+        if (resultflags.HasInstructionSet(InstructionSet_AVXVNNIINT_V512) && !resultflags.HasInstructionSet(InstructionSet_AVX10v2))
+            resultflags.RemoveInstructionSet(InstructionSet_AVXVNNIINT_V512);
         if (resultflags.HasInstructionSet(InstructionSet_Vector128) && !resultflags.HasInstructionSet(InstructionSet_X86Base))
             resultflags.RemoveInstructionSet(InstructionSet_Vector128);
         if (resultflags.HasInstructionSet(InstructionSet_Vector256) && !resultflags.HasInstructionSet(InstructionSet_AVX))
@@ -565,6 +573,10 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins
             resultflags.RemoveInstructionSet(InstructionSet_WAITPKG);
         if (resultflags.HasInstructionSet(InstructionSet_X86Serialize) && !resultflags.HasInstructionSet(InstructionSet_X86Base))
             resultflags.RemoveInstructionSet(InstructionSet_X86Serialize);
+        if (resultflags.HasInstructionSet(InstructionSet_AVXVNNIINT) && !resultflags.HasInstructionSet(InstructionSet_AVX2))
+            resultflags.RemoveInstructionSet(InstructionSet_AVXVNNIINT);
+        if (resultflags.HasInstructionSet(InstructionSet_AVXVNNIINT_V512) && !resultflags.HasInstructionSet(InstructionSet_AVX10v2))
+            resultflags.RemoveInstructionSet(InstructionSet_AVXVNNIINT_V512);
         if (resultflags.HasInstructionSet(InstructionSet_Vector128) && !resultflags.HasInstructionSet(InstructionSet_X86Base))
             resultflags.RemoveInstructionSet(InstructionSet_Vector128);
         if (resultflags.HasInstructionSet(InstructionSet_Vector256) && !resultflags.HasInstructionSet(InstructionSet_AVX))
@@ -747,6 +759,10 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet)
             return "VectorT256";
         case InstructionSet_VectorT512 :
             return "VectorT512";
+        case InstructionSet_AVXVNNIINT :
+            return "AVXVNNIINT";
+        case InstructionSet_AVXVNNIINT_V512 :
+            return "AVXVNNIINT_V512";
 #endif // TARGET_AMD64
 #ifdef TARGET_X86
         case InstructionSet_X86Base :
@@ -805,6 +821,10 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet)
             return "VectorT256";
         case InstructionSet_VectorT512 :
             return "VectorT512";
+        case InstructionSet_AVXVNNIINT :
+            return "AVXVNNIINT";
+        case InstructionSet_AVXVNNIINT_V512 :
+            return "AVXVNNIINT_V512";
 #endif // TARGET_X86
 
         default:
@@ -909,6 +929,10 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst
         case READYTORUN_INSTRUCTION_VectorT128: return InstructionSet_VectorT128;
         case READYTORUN_INSTRUCTION_VectorT256: return InstructionSet_VectorT256;
         case READYTORUN_INSTRUCTION_VectorT512: return InstructionSet_VectorT512;
+        case READYTORUN_INSTRUCTION_AvxVnniInt8: return InstructionSet_AVXVNNIINT;
+        case READYTORUN_INSTRUCTION_AvxVnniInt8_V512: return InstructionSet_AVXVNNIINT_V512;
+        case READYTORUN_INSTRUCTION_AvxVnniInt16: return InstructionSet_AVXVNNIINT;
+        case READYTORUN_INSTRUCTION_AvxVnniInt16_V512: return InstructionSet_AVXVNNIINT_V512;
 #endif // TARGET_AMD64
 #ifdef TARGET_X86
         case READYTORUN_INSTRUCTION_X86Base: return InstructionSet_X86Base;
@@ -974,6 +998,10 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst
         case READYTORUN_INSTRUCTION_VectorT128: return InstructionSet_VectorT128;
         case READYTORUN_INSTRUCTION_VectorT256: return InstructionSet_VectorT256;
         case READYTORUN_INSTRUCTION_VectorT512: return InstructionSet_VectorT512;
+        case READYTORUN_INSTRUCTION_AvxVnniInt8: return InstructionSet_AVXVNNIINT;
+        case READYTORUN_INSTRUCTION_AvxVnniInt8_V512: return InstructionSet_AVXVNNIINT_V512;
+        case READYTORUN_INSTRUCTION_AvxVnniInt16: return InstructionSet_AVXVNNIINT;
+        case READYTORUN_INSTRUCTION_AvxVnniInt16_V512: return InstructionSet_AVXVNNIINT_V512;
 #endif // TARGET_X86
 
         default:

src/coreclr/inc/jiteeversionguid.h

@@ -37,11 +37,11 @@
 
 #include <minipal/guid.h>
 
-constexpr GUID JITEEVersionIdentifier = { /* ce8cef5e-261f-469a-b599-9f3f3e8b2448 */
-    0xce8cef5e,
-    0x261f,
-    0x469a,
-    {0xb5, 0x99, 0x9f, 0x3f, 0x3e, 0x8b, 0x24, 0x48}
+constexpr GUID JITEEVersionIdentifier = { /* 5c7eb9f1-a9cb-4a35-aea6-ae93d1f54c56 */
+    0x5c7eb9f1,
+    0xa9cb,
+    0x4a35,
+    {0xae, 0xa6, 0xae, 0x93, 0xd1, 0xf5, 0x4c, 0x56}
   };
 
 #endif // JIT_EE_VERSIONING_GUID_H

src/coreclr/inc/readytoruninstructionset.h

@@ -67,6 +67,10 @@ enum ReadyToRunInstructionSet
     READYTORUN_INSTRUCTION_Zba=57,
     READYTORUN_INSTRUCTION_Zbb=58,
     READYTORUN_INSTRUCTION_Sve2=59,
+    READYTORUN_INSTRUCTION_AvxVnniInt8=60,
+    READYTORUN_INSTRUCTION_AvxVnniInt8_V512=61,
+    READYTORUN_INSTRUCTION_AvxVnniInt16=62,
+    READYTORUN_INSTRUCTION_AvxVnniInt16_V512=63,
     READYTORUN_INSTRUCTION_Aes_V256=64,
     READYTORUN_INSTRUCTION_Aes_V512=65,
     READYTORUN_INSTRUCTION_AvxIfma=66,

src/coreclr/jit/compiler.h

@@ -9604,6 +9604,13 @@ class Compiler
         return false;
     }
 
+#ifdef FEATURE_HW_INTRINSICS
+    CORINFO_InstructionSet lookupInstructionSet(const char* className);
+    CORINFO_InstructionSet lookupIsa(const char* className,
+                                     const char* innerEnclosingClassName,
+                                     const char* outerEnclosingClassName);
+#endif // FEATURE_HW_INTRINSICS
+
 #ifdef DEBUG
     // Answer the question: Is a particular ISA supported?
     // Use this api when asking the question so that future

src/coreclr/jit/emitxarch.cpp

@@ -90,6 +90,17 @@ bool emitter::IsApxOnlyInstruction(instruction ins)
     return (ins >= FIRST_APX_INSTRUCTION) && (ins <= LAST_APX_INSTRUCTION);
 }
 
+bool emitter::IsAVXVNNIFamilyInstruction(instruction ins)
+{
+    return (ins >= FIRST_AVXVNNI_INSTRUCTION && ins <= LAST_AVXVNNI_INSTRUCTION) || IsAVXVNNIINTInstruction(ins);
+}
+
+bool emitter::IsAVXVNNIINTInstruction(instruction ins)
+{
+    return (ins >= FIRST_AVXVNNIINT8_INSTRUCTION && ins <= LAST_AVXVNNIINT8_INSTRUCTION) ||
+           (ins >= FIRST_AVXVNNIINT16_INSTRUCTION && ins <= LAST_AVXVNNIINT16_INSTRUCTION);
+}
+
 bool emitter::Is3OpRmwInstruction(instruction ins)
 {
     switch (ins)
@@ -113,7 +124,7 @@ bool emitter::Is3OpRmwInstruction(instruction ins)
         default:
         {
             return ((ins >= FIRST_FMA_INSTRUCTION) && (ins <= LAST_FMA_INSTRUCTION)) ||
-                   ((ins >= FIRST_AVXVNNI_INSTRUCTION) && (ins <= LAST_AVXVNNI_INSTRUCTION)) ||
+                   (IsAVXVNNIFamilyInstruction(ins)) ||
                    ((ins >= FIRST_AVXIFMA_INSTRUCTION) && (ins <= LAST_AVXIFMA_INSTRUCTION));
         }
     }
@@ -278,6 +289,23 @@ bool emitter::IsVexEncodableInstruction(instruction ins) const
             return emitComp->compSupportsHWIntrinsic(InstructionSet_AVXVNNI);
         }
 
+        case INS_vpdpwsud:
+        case INS_vpdpwsuds:
+        case INS_vpdpwusd:
+        case INS_vpdpwusds:
+        case INS_vpdpwuud:
+        case INS_vpdpwuuds:
+        case INS_vpdpbssd:
+        case INS_vpdpbssds:
+        case INS_vpdpbsud:
+        case INS_vpdpbsuds:
+        case INS_vpdpbuud:
+        case INS_vpdpbuuds:
+        {
+            // Vex versions of AvxVnniInt8 + AvxVnniInt16
+            return emitComp->compSupportsHWIntrinsic(InstructionSet_AVXVNNIINT);
+        }
+
         case INS_vpmadd52huq:
         case INS_vpmadd52luq:
         {
@@ -325,6 +353,23 @@ bool emitter::IsEvexEncodableInstruction(instruction ins) const
             return emitComp->compSupportsHWIntrinsic(InstructionSet_AES_V512);
         }
 
+        case INS_vpdpwsud:
+        case INS_vpdpwsuds:
+        case INS_vpdpwusd:
+        case INS_vpdpwusds:
+        case INS_vpdpwuud:
+        case INS_vpdpwuuds:
+        case INS_vpdpbssd:
+        case INS_vpdpbssds:
+        case INS_vpdpbsud:
+        case INS_vpdpbsuds:
+        case INS_vpdpbuud:
+        case INS_vpdpbuuds:
+        {
+            // Evex versions of AvxVnniInt8 + AvxVnniInt16 will be supported
+            return emitComp->compSupportsHWIntrinsic(InstructionSet_AVXVNNIINT_V512);
+        }
+
         case INS_vpdpbusd:
         case INS_vpdpwssd:
         case INS_vpdpbusds:
@@ -2928,7 +2973,9 @@ emitter::code_t emitter::emitExtractEvexPrefix(instruction ins, code_t& code) co
         if (sizePrefix == 0)
         {
             // no simd prefix for EVEX2 - AVX10.2 and above
-            assert(emitComp->compIsaSupportedDebugOnly(InstructionSet_AVX10v2));
+            assert(emitComp->compIsaSupportedDebugOnly(InstructionSet_AVX10v2) ||
+                   emitComp->compIsaSupportedDebugOnly(InstructionSet_AVXVNNIINT) ||
+                   emitComp->compIsaSupportedDebugOnly(InstructionSet_AVXVNNIINT_V512));
         }
         else if (isPrefix(sizePrefix))
         {
@@ -3139,7 +3186,14 @@ emitter::code_t emitter::emitExtractVexPrefix(instruction ins, code_t& code) con
         // check for a prefix in the 11 position
         BYTE sizePrefix = (code >> 16) & 0xFF;
 
-        if ((sizePrefix != 0) && isPrefix(sizePrefix))
+        if (sizePrefix == 0)
+        {
+            // no simd prefix for Avx-Vnni-Int* ISAs subset of instructions
+            // INS_vpdpbuud[,s], INS_vpdpwuud[,s]
+            assert(emitComp->compIsaSupportedDebugOnly(InstructionSet_AVXVNNIINT) ||
+                   emitComp->compIsaSupportedDebugOnly(InstructionSet_AVXVNNIINT_V512));
+        }
+        else if (isPrefix(sizePrefix))
         {
             // 'pp' bits in byte2 of VEX prefix allows us to encode SIMD size prefixes as two bits
             //
@@ -3209,23 +3263,27 @@ emitter::code_t emitter::emitExtractVexPrefix(instruction ins, code_t& code) con
                     unreached();
                 }
             }
+        }
+        else
+        {
+            unreached();
+        }
 
-            // Now the byte in the 22 position must be an escape byte 0F
-            leadingBytes = check;
-            assert(leadingBytes == 0x0F);
+        // Now the byte in the 22 position must be an escape byte 0F
+        leadingBytes = check;
+        assert(leadingBytes == 0x0F);
 
-            // Get rid of both sizePrefix and escape byte
-            code &= 0x0000FFFFLL;
+        // Get rid of both sizePrefix and escape byte
+        code &= 0x0000FFFFLL;
 
-            // Check the byte in the 33 position to see if it is 3A or 38.
-            // In such a case escape bytes must be 0x0F3A or 0x0F38
-            check = code & 0xFF;
+        // Check the byte in the 33 position to see if it is 3A or 38.
+        // In such a case escape bytes must be 0x0F3A or 0x0F38
+        check = code & 0xFF;
 
-            if ((check == 0x3A) || (check == 0x38))
-            {
-                leadingBytes = (leadingBytes << 8) | check;
-                code &= 0x0000FF00LL;
-            }
+        if ((check == 0x3A) || (check == 0x38))
+        {
+            leadingBytes = (leadingBytes << 8) | check;
+            code &= 0x0000FF00LL;
         }
     }
     else
@@ -4378,7 +4436,7 @@ bool emitter::EncodedBySSE38orSSE3A(instruction ins) const
 
 #if defined(DEBUG)
     insCode = (insCode >> 16) & 0xFF;
-    assert((insCode == 0x66) || (insCode == 0xF2) || (insCode == 0xF3));
+    assert((insCode == 0x00) || (insCode == 0x66) || (insCode == 0xF2) || (insCode == 0xF3));
 #endif // DEBUG
 
     return true;
@@ -18083,7 +18141,8 @@ ssize_t emitter::TryEvexCompressDisp8Byte(instrDesc* id, ssize_t dsp, bool* dspI
     {
         case INS_TT_FULL:
         {
-            assert(inputSize == 4 || inputSize == 8);
+            instruction ins = id->idIns();
+            assert((inputSize == 4 || inputSize == 8) || IsAVXVNNIINTInstruction(ins));
             if (HasEmbeddedBroadcast(id))
             {
                 // N = input size in bytes

src/coreclr/jit/emitxarch.h

@@ -121,6 +121,8 @@ static bool IsSSEOrAVXInstruction(instruction ins);
 static bool IsAVXOnlyInstruction(instruction ins);
 static bool IsAvx512OnlyInstruction(instruction ins);
 static bool IsKMOVInstruction(instruction ins);
+static bool IsAVXVNNIFamilyInstruction(instruction ins);
+static bool IsAVXVNNIINTInstruction(instruction ins);
 static bool Is3OpRmwInstruction(instruction ins);
 static bool IsBMIInstruction(instruction ins);
 static bool IsKInstruction(instruction ins);

src/coreclr/jit/hwintrinsic.cpp

@@ -930,6 +930,8 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = {
     { NI_Illegal, NI_Illegal },                                 //      VectorT128
     { NI_Illegal, NI_Illegal },                                 //      VectorT256
     { NI_Illegal, NI_Illegal },                                 //      VectorT512
+    { FIRST_NI_AVXVNNIINT, LAST_NI_AVXVNNIINT },                // AVXVNNIINT
+    { FIRST_NI_AVXVNNIINT_V512, LAST_NI_AVXVNNIINT_V512 },      // AVXVNNIINT_V512
 
     { FIRST_NI_X86Base_X64, LAST_NI_X86Base_X64 },              // X86Base_X64
     { FIRST_NI_SSE42_X64, LAST_NI_SSE42_X64 },                  // SSE42_X64
@@ -1180,7 +1182,7 @@ NamedIntrinsic HWIntrinsicInfo::lookupId(Compiler*         comp,
         return NI_Illegal;
     }
 
-    CORINFO_InstructionSet isa = lookupIsa(className, innerEnclosingClassName, outerEnclosingClassName);
+    CORINFO_InstructionSet isa = comp->lookupIsa(className, innerEnclosingClassName, outerEnclosingClassName);
 
     if (isa == InstructionSet_ILLEGAL)
     {