Arm64: use FEAT_CSSC cnt/ctz for PopCount and TrailingZeroCount

src/coreclr/inc/clrconfigvalues.h

@@ -705,6 +705,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sha1,              W("EnableArm64Sh
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sha256,            W("EnableArm64Sha256"),         1, "Allows Arm64 Sha256+ hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Rcpc,              W("EnableArm64Rcpc"),           1, "Allows Arm64 Rcpc+ hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Rcpc2,             W("EnableArm64Rcpc2"),          1, "Allows Arm64 Rcpc2+ hardware intrinsics to be disabled")
+RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Cssc,              W("EnableArm64Cssc"),           0, "Allows Arm64 Cssc+ hardware intrinsics to be disabled") // TODO: enable when CI gets armv8.9+ hardware
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sve,               W("EnableArm64Sve"),            1, "Allows Arm64 SVE+ hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sve2,              W("EnableArm64Sve2"),           1, "Allows Arm64 SVE2+ hardware intrinsics to be disabled")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableArm64Sha3,              W("EnableArm64Sha3"),           1, "Allows Arm64 Sha3+ hardware intrinsics to be disabled")

src/coreclr/inc/corinfoinstructionset.h

@@ -40,21 +40,22 @@ enum CORINFO_InstructionSet
     InstructionSet_SveAes=21,
     InstructionSet_SveSha3=22,
     InstructionSet_SveSm4=23,
-    InstructionSet_ArmBase_Arm64=24,
-    InstructionSet_AdvSimd_Arm64=25,
-    InstructionSet_Aes_Arm64=26,
-    InstructionSet_Crc32_Arm64=27,
-    InstructionSet_Dp_Arm64=28,
-    InstructionSet_Rdm_Arm64=29,
-    InstructionSet_Sha1_Arm64=30,
-    InstructionSet_Sha256_Arm64=31,
-    InstructionSet_Sve_Arm64=32,
-    InstructionSet_Sve2_Arm64=33,
-    InstructionSet_Sha3_Arm64=34,
-    InstructionSet_Sm4_Arm64=35,
-    InstructionSet_SveAes_Arm64=36,
-    InstructionSet_SveSha3_Arm64=37,
-    InstructionSet_SveSm4_Arm64=38,
+    InstructionSet_Cssc=24,
+    InstructionSet_ArmBase_Arm64=25,
+    InstructionSet_AdvSimd_Arm64=26,
+    InstructionSet_Aes_Arm64=27,
+    InstructionSet_Crc32_Arm64=28,
+    InstructionSet_Dp_Arm64=29,
+    InstructionSet_Rdm_Arm64=30,
+    InstructionSet_Sha1_Arm64=31,
+    InstructionSet_Sha256_Arm64=32,
+    InstructionSet_Sve_Arm64=33,
+    InstructionSet_Sve2_Arm64=34,
+    InstructionSet_Sha3_Arm64=35,
+    InstructionSet_Sm4_Arm64=36,
+    InstructionSet_SveAes_Arm64=37,
+    InstructionSet_SveSha3_Arm64=38,
+    InstructionSet_SveSm4_Arm64=39,
 #endif // TARGET_ARM64
 #ifdef TARGET_RISCV64
     InstructionSet_RiscV64Base=1,
@@ -745,6 +746,8 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet)
             return "SveSm4";
         case InstructionSet_SveSm4_Arm64 :
             return "SveSm4_Arm64";
+        case InstructionSet_Cssc :
+            return "Cssc";
 #endif // TARGET_ARM64
 #ifdef TARGET_RISCV64
         case InstructionSet_RiscV64Base :
@@ -958,6 +961,7 @@ inline CORINFO_InstructionSet InstructionSetFromR2RInstructionSet(ReadyToRunInst
         case READYTORUN_INSTRUCTION_SveAes: return InstructionSet_SveAes;
         case READYTORUN_INSTRUCTION_SveSha3: return InstructionSet_SveSha3;
         case READYTORUN_INSTRUCTION_SveSm4: return InstructionSet_SveSm4;
+        case READYTORUN_INSTRUCTION_Cssc: return InstructionSet_Cssc;
 #endif // TARGET_ARM64
 #ifdef TARGET_RISCV64
         case READYTORUN_INSTRUCTION_RiscV64Base: return InstructionSet_RiscV64Base;

src/coreclr/inc/jiteeversionguid.h

@@ -37,11 +37,11 @@
 
 #include <minipal/guid.h>
 
-constexpr GUID JITEEVersionIdentifier = { /* 58dd28fc-38f0-4782-989a-25e2d721a98b */
-    0x58dd28fc,
-    0x38f0,
-    0x4782,
-    {0x98, 0x9a, 0x25, 0xe2, 0xd7, 0x21, 0xa9, 0x8b}
+constexpr GUID JITEEVersionIdentifier = { /* 2ff14624-7bec-4037-b0c0-c75d03d6433c */
+    0x2ff14624,
+    0x7bec,
+    0x4037,
+    {0xb0, 0xc0, 0xc7, 0x5d, 0x03, 0xd6, 0x43, 0x3c}
   };
 
 #endif // JIT_EE_VERSIONING_GUID_H

src/coreclr/inc/readytoruninstructionset.h

@@ -100,6 +100,7 @@ enum ReadyToRunInstructionSet
     READYTORUN_INSTRUCTION_SveSm4=90,
     READYTORUN_INSTRUCTION_WasmBase=91,
     READYTORUN_INSTRUCTION_PackedSimd=92,
+    READYTORUN_INSTRUCTION_Cssc=93,
 
 };
 

src/coreclr/jit/codegenarm64test.cpp

@@ -912,12 +912,16 @@ void CodeGen::genArm64EmitterUnitTestsGeneral()
     theEmitter->emitIns_R_R(INS_rev, EA_8BYTE, REG_R4, REG_R15);
     theEmitter->emitIns_R_R(INS_rev16, EA_8BYTE, REG_R5, REG_R0);
     theEmitter->emitIns_R_R(INS_rev32, EA_8BYTE, REG_R6, REG_R1);
+    theEmitter->emitIns_R_R(INS_cnt, EA_8BYTE, REG_R3, REG_R14); // FEAT_CSSC
+    theEmitter->emitIns_R_R(INS_ctz, EA_8BYTE, REG_R4, REG_R15); // FEAT_CSSC
 
     theEmitter->emitIns_R_R(INS_cls, EA_4BYTE, REG_R7, REG_R2);
     theEmitter->emitIns_R_R(INS_clz, EA_4BYTE, REG_R8, REG_R3);
     theEmitter->emitIns_R_R(INS_rbit, EA_4BYTE, REG_R9, REG_R4);
     theEmitter->emitIns_R_R(INS_rev, EA_4BYTE, REG_R10, REG_R5);
     theEmitter->emitIns_R_R(INS_rev16, EA_4BYTE, REG_R11, REG_R6);
+    theEmitter->emitIns_R_R(INS_cnt, EA_4BYTE, REG_R9, REG_R4);  // FEAT_CSSC
+    theEmitter->emitIns_R_R(INS_ctz, EA_4BYTE, REG_R10, REG_R5); // FEAT_CSSC
 
     //
     // R_I

src/coreclr/jit/codegenarmarch.cpp

@@ -717,14 +717,23 @@ void CodeGen::genIntrinsic(GenTreeIntrinsic* treeNode)
         {
             genConsumeOperands(treeNode->AsOp());
 
-            regNumber tmpReg = internalRegisters.GetSingle(treeNode);
             regNumber srcReg = srcNode->GetRegNum();
             regNumber dstReg = treeNode->GetRegNum();
 
-            assert(genIsValidFloatReg(tmpReg));
             assert(genIsValidIntReg(srcReg));
             assert(genIsValidIntReg(dstReg));
 
+            if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_Cssc))
+            {
+                // FEAT_CSSC provides a scalar cnt operating directly on general registers.
+                GetEmitter()->emitIns_R_R(INS_cnt, emitActualTypeSize(srcNode), dstReg, srcReg, INS_OPTS_NONE);
+                break;
+            }
+
+            regNumber tmpReg = internalRegisters.GetSingle(treeNode);
+
+            assert(genIsValidFloatReg(tmpReg));
+
             emitAttr attr = emitTypeSize(srcNode->TypeGet());
 
             if (attr != EA_8BYTE)
@@ -748,6 +757,13 @@ void CodeGen::genIntrinsic(GenTreeIntrinsic* treeNode)
             assert(genIsValidIntReg(srcReg));
             assert(genIsValidIntReg(dstReg));
 
+            if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_Cssc))
+            {
+                // FEAT_CSSC provides a scalar ctz operating directly on general registers.
+                GetEmitter()->emitIns_R_R(INS_ctz, emitActualTypeSize(srcNode), dstReg, srcReg, INS_OPTS_NONE);
+                break;
+            }
+
             GetEmitter()->emitIns_R_R(INS_rbit, emitActualTypeSize(srcNode), dstReg, srcReg, INS_OPTS_NONE);
             GetEmitter()->emitIns_R_R(INS_clz, emitActualTypeSize(srcNode), dstReg, dstReg, INS_OPTS_NONE);
             break;

src/coreclr/jit/emitarm64.cpp

@@ -4596,14 +4596,11 @@ void emitter::emitIns_R_R(instruction     ins,
                 fmt = IF_DV_2M;
                 break;
             }
-            if (ins == INS_cnt)
-            {
-                // Doesn't have general register version(s)
-                break;
-            }
+            // INS_cnt on general registers requires FEAT_CSSC; fall through to the DR_2G encoding.
 
             FALLTHROUGH;
 
+        case INS_ctz:
         case INS_rev:
             assert(insOptsNone(opt));
             assert(isGeneralRegister(reg1));

src/coreclr/jit/hwintrinsic.cpp

@@ -997,6 +997,7 @@ static const HWIntrinsicIsaRange hwintrinsicIsaRangeArray[] = {
     { NI_Illegal, NI_Illegal },                                 //      SveAes
     { FIRST_NI_SveSha3, LAST_NI_SveSha3 },                      // SveSha3
     { NI_Illegal, NI_Illegal },                                 //      SveSm4
+    { NI_Illegal, NI_Illegal },                                 //      Cssc
     { FIRST_NI_ArmBase_Arm64, LAST_NI_ArmBase_Arm64 },          // ArmBase_Arm64
     { FIRST_NI_AdvSimd_Arm64, LAST_NI_AdvSimd_Arm64 },          // AdvSimd_Arm64
     { NI_Illegal, NI_Illegal },                                 //      Aes_Arm64

src/coreclr/jit/instrsarm64.h

@@ -816,6 +816,10 @@ INST2(rbit,        "rbit",         0,      IF_EN2L,   0x5AC00000,  0x2E605800)
                                    //  rbit    Rd,Rm                DR_2G  X101101011000000 000000nnnnnddddd   5AC0 0000   Rd Rn    (general)
                                    //  rbit    Vd,Vn                DV_2M  0Q10111001100000 010110nnnnnddddd   2E60 5800   Vd,Vn    (vector)
 
+INST2(cnt,         "cnt",          0,      IF_EN2L,   0x5AC01C00,  0x0E205800)
+                                   //  cnt     Rd,Rm                DR_2G  X101101011000000 000111nnnnnddddd   5AC0 1C00   Rd Rn    (general, FEAT_CSSC)
+                                   //  cnt     Vd,Vn                DV_2M  0Q00111000100000 010110nnnnnddddd   0E20 5800   Vd,Vn    (vector)
+
 INST2(rev16,       "rev16",        0,      IF_EN2L,   0x5AC00400,  0x0E201800)
                                    //  rev16   Rd,Rm                DR_2G  X101101011000000 000001nnnnnddddd   5AC0 0400   Rd Rn    (general)
                                    //  rev16   Vd,Vn                DV_2M  0Q001110XX100000 000110nnnnnddddd   0E20 1800   Vd,Vn    (vector)
@@ -1716,8 +1720,8 @@ INST1(bif,         "bif",          0,      IF_DV_3C,  0x2EE01C00)
 INST1(addv,        "addv",         0,      IF_DV_2T,  0x0E31B800)
                                    //  addv    Vd,Vn                DV_2T  0Q001110XX110001 101110nnnnnddddd   0E31 B800   Vd,Vn      (vector)
 
-INST1(cnt,         "cnt",          0,      IF_DV_2M,  0x0E205800)
-                                   //  cnt     Vd,Vn                DV_2M  0Q00111000100000 010110nnnnnddddd   0E20 5800   Vd,Vn      (vector)
+INST1(ctz,         "ctz",          0,      IF_DR_2G,  0x5AC01800)
+                                   //  ctz     Rd,Rn                DR_2G  X101101011000000 000110nnnnnddddd   5AC0 1800   Rd Rn      (general, FEAT_CSSC)
 
 INST1(not,         "not",          0,      IF_DV_2M,  0x2E205800)
                                    //  not     Vd,Vn                DV_2M  0Q10111000100000 010110nnnnnddddd   2E20 5800   Vd,Vn      (vector)

src/coreclr/jit/lsraarm64.cpp

@@ -985,8 +985,11 @@ int LinearScan::BuildNode(GenTree* tree)
                 {
                     assert(varTypeIsIntegral(tree->gtGetOp1()));
 
-                    // We need a SIMD register to execute popcnt
-                    buildInternalFloatRegisterDefForNode(tree, allSIMDRegs());
+                    // Without FEAT_CSSC we need a SIMD register to execute popcnt
+                    if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_Cssc))
+                    {
+                        buildInternalFloatRegisterDefForNode(tree, allSIMDRegs());
+                    }
 
                     BuildUse(tree->gtGetOp1());
                     buildInternalRegisterUses();

src/coreclr/tools/Common/Compiler/HardwareIntrinsicHelpers.cs

@@ -232,6 +232,7 @@ private static class Arm64IntrinsicConstants
             public const int SveAes = (1 << 13);
             public const int SveSha3 = (1 << 14);
             public const int SveSm4 = (1 << 15);
+            public const int Cssc = (1 << 16);
 
             public static void AddToBuilder(InstructionSetSupportBuilder builder, int flags)
             {
@@ -267,6 +268,8 @@ public static void AddToBuilder(InstructionSetSupportBuilder builder, int flags)
                     builder.AddSupportedInstructionSet("sve_sha3");
                 if ((flags & SveSm4) != 0)
                     builder.AddSupportedInstructionSet("sve_sm4");
+                if ((flags & Cssc) != 0)
+                    builder.AddSupportedInstructionSet("cssc");
             }
 
             public static int FromInstructionSet(InstructionSet instructionSet)
@@ -310,6 +313,7 @@ public static int FromInstructionSet(InstructionSet instructionSet)
                     InstructionSet.ARM64_SveSha3_Arm64 => SveSha3,
                     InstructionSet.ARM64_SveSm4 => SveSm4,
                     InstructionSet.ARM64_SveSm4_Arm64 => SveSm4,
+                    InstructionSet.ARM64_Cssc => Cssc,
 
                     // Vector<T> Sizes
                     InstructionSet.ARM64_VectorT128 => 0,

src/coreclr/tools/Common/InstructionSetHelpers.cs

@@ -301,6 +301,7 @@ public static InstructionSetSupport ConfigureInstructionSetSupport(string instru
             {
                 optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("aes");
                 optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("crc");
+                optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("cssc");
                 optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("dotprod");
                 optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("lse");
                 optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("rcpc");

src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSet.cs

@@ -103,5 +103,6 @@ public enum ReadyToRunInstructionSet
         SveSm4 = 90,
         WasmBase = 91,
         PackedSimd = 92,
+        Cssc = 93,
     }
 }

src/coreclr/tools/Common/Internal/Runtime/ReadyToRunInstructionSetHelper.cs

@@ -60,6 +60,7 @@ public static class ReadyToRunInstructionSetHelper
                         case InstructionSet.ARM64_SveSha3_Arm64: return ReadyToRunInstructionSet.SveSha3;
                         case InstructionSet.ARM64_SveSm4: return ReadyToRunInstructionSet.SveSm4;
                         case InstructionSet.ARM64_SveSm4_Arm64: return ReadyToRunInstructionSet.SveSm4;
+                        case InstructionSet.ARM64_Cssc: return ReadyToRunInstructionSet.Cssc;
 
                         default: throw new Exception("Unknown instruction set");
                     }

src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs

@@ -40,6 +40,7 @@ public enum InstructionSet
         ARM64_SveAes = InstructionSet_ARM64.SveAes,
         ARM64_SveSha3 = InstructionSet_ARM64.SveSha3,
         ARM64_SveSm4 = InstructionSet_ARM64.SveSm4,
+        ARM64_Cssc = InstructionSet_ARM64.Cssc,
         ARM64_ArmBase_Arm64 = InstructionSet_ARM64.ArmBase_Arm64,
         ARM64_AdvSimd_Arm64 = InstructionSet_ARM64.AdvSimd_Arm64,
         ARM64_Aes_Arm64 = InstructionSet_ARM64.Aes_Arm64,
@@ -182,21 +183,22 @@ public enum InstructionSet_ARM64
         SveAes = 21,
         SveSha3 = 22,
         SveSm4 = 23,
-        ArmBase_Arm64 = 24,
-        AdvSimd_Arm64 = 25,
-        Aes_Arm64 = 26,
-        Crc32_Arm64 = 27,
-        Dp_Arm64 = 28,
-        Rdm_Arm64 = 29,
-        Sha1_Arm64 = 30,
-        Sha256_Arm64 = 31,
-        Sve_Arm64 = 32,
-        Sve2_Arm64 = 33,
-        Sha3_Arm64 = 34,
-        Sm4_Arm64 = 35,
-        SveAes_Arm64 = 36,
-        SveSha3_Arm64 = 37,
-        SveSm4_Arm64 = 38,
+        Cssc = 24,
+        ArmBase_Arm64 = 25,
+        AdvSimd_Arm64 = 26,
+        Aes_Arm64 = 27,
+        Crc32_Arm64 = 28,
+        Dp_Arm64 = 29,
+        Rdm_Arm64 = 30,
+        Sha1_Arm64 = 31,
+        Sha256_Arm64 = 32,
+        Sve_Arm64 = 33,
+        Sve2_Arm64 = 34,
+        Sha3_Arm64 = 35,
+        Sm4_Arm64 = 36,
+        SveAes_Arm64 = 37,
+        SveSha3_Arm64 = 38,
+        SveSm4_Arm64 = 39,
     }
 
     public enum InstructionSet_RiscV64
@@ -1149,6 +1151,7 @@ public static IEnumerable<InstructionSetInfo> ArchitectureToValidInstructionSets
                     yield return new InstructionSetInfo("sve_aes", "SveAes", InstructionSet.ARM64_SveAes, true);
                     yield return new InstructionSetInfo("sve_sha3", "SveSha3", InstructionSet.ARM64_SveSha3, true);
                     yield return new InstructionSetInfo("sve_sm4", "SveSm4", InstructionSet.ARM64_SveSm4, true);
+                    yield return new InstructionSetInfo("cssc", "", InstructionSet.ARM64_Cssc, true);
                     break;
 
                 case TargetArchitecture.RiscV64:

src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt

@@ -26,7 +26,7 @@
 ; DO NOT CHANGE R2R NUMERIC VALUES OF THE EXISTING SETS. Changing R2R numeric values definitions would be R2R format breaking change.
 
 ; The ISA definitions should also be mapped to `hwintrinsicIsaRangeArray` in hwintrinsic.cpp.
-; NEXT_AVAILABLE_R2R_BIT = 93
+; NEXT_AVAILABLE_R2R_BIT = 94
 
 ; Definition of X86 instruction sets
 definearch         ,X86   ,32Bit                ,X64, X64, X86
@@ -232,6 +232,7 @@ instructionset     ,ARM64 ,Sm4                   ,           ,87 ,Sm4
 instructionset     ,ARM64 ,SveAes                ,           ,88 ,SveAes                ,sve_aes
 instructionset     ,ARM64 ,SveSha3               ,           ,89 ,SveSha3               ,sve_sha3
 instructionset     ,ARM64 ,SveSm4                ,           ,90 ,SveSm4                ,sve_sm4
+instructionset     ,ARM64 ,                      ,Cssc       ,93 ,Cssc                  ,cssc
 
 instructionset64bit,ARM64 ,ArmBase
 instructionset64bit,ARM64 ,AdvSimd

src/coreclr/vm/codeman.cpp

@@ -1687,6 +1687,11 @@ void EEJitManager::SetCpuInfo()
         CPUCompileFlags.Set(InstructionSet_Rcpc2);
     }
 
+    if (((cpuFeatures & ARM64IntrinsicConstants_Cssc) != 0) && CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_EnableArm64Cssc))
+    {
+        CPUCompileFlags.Set(InstructionSet_Cssc);
+    }
+
     if (((cpuFeatures & ARM64IntrinsicConstants_Crc32) != 0) && CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_EnableArm64Crc32))
     {
         CPUCompileFlags.Set(InstructionSet_Crc32);

src/native/minipal/cpufeatures.c

@@ -83,6 +83,9 @@
 #ifndef HWCAP2_SVESM4
 #define HWCAP2_SVESM4   (1 << 6)
 #endif
+#ifndef HWCAP2_CSSC
+#define HWCAP2_CSSC   (1UL << 34)
+#endif
 
 #endif
 
@@ -566,6 +569,9 @@ int minipal_getcpufeatures(void)
     if (hwCap2 & HWCAP2_SVESM4)
         result |= ARM64IntrinsicConstants_SveSm4;
 
+    if (hwCap2 & HWCAP2_CSSC)
+        result |= ARM64IntrinsicConstants_Cssc;
+
 #else // !HAVE_AUXV_HWCAP_H
 
 #if HAVE_SYSCTLBYNAME
@@ -636,6 +642,9 @@ int minipal_getcpufeatures(void)
 
     if ((sysctlbyname("hw.optional.arm.FEAT_SVE_SM4", &valueFromSysctl, &sz, NULL, 0) == 0) && (valueFromSysctl != 0))
         result |= ARM64IntrinsicConstants_SveSm4;
+
+    if ((sysctlbyname("hw.optional.arm.FEAT_CSSC", &valueFromSysctl, &sz, NULL, 0) == 0) && (valueFromSysctl != 0))
+        result |= ARM64IntrinsicConstants_Cssc;
 #endif // HAVE_SYSCTLBYNAME
 #endif // HAVE_AUXV_HWCAP_H
 #endif // HOST_UNIX
@@ -718,6 +727,8 @@ int minipal_getcpufeatures(void)
     {
         result |= ARM64IntrinsicConstants_SveSm4;
     }
+
+    // TODO: IsProcessorFeaturePresent doesn't support CSSC yet.
 #endif // HOST_WINDOWS
 
 #endif // HOST_ARM64

src/native/minipal/cpufeatures.h

@@ -50,6 +50,7 @@
 #define ARM64IntrinsicConstants_SveAes (1 << 13)
 #define ARM64IntrinsicConstants_SveSha3 (1 << 14)
 #define ARM64IntrinsicConstants_SveSm4 (1 << 15)
+#define ARM64IntrinsicConstants_Cssc (1 << 16)
 
 #include <assert.h>