diff --git a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_fp8.h b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_fp8.h index 18823f16b8a2..fa3e7b4dcac6 100644 --- a/projects/clr/hipamd/include/hip/amd_detail/amd_hip_fp8.h +++ b/projects/clr/hipamd/include/hip/amd_detail/amd_hip_fp8.h @@ -30,6 +30,8 @@ #ifndef _HIP_INCLUDE_HIP_AMD_DETAIL_HIP_FP8_H_ #define _HIP_INCLUDE_HIP_AMD_DETAIL_HIP_FP8_H_ +#include +#include "device_library_decls.h" #if (defined(__gfx942__) || defined(__gfx1200__) || defined(__gfx1201__) || \ defined(__gfx950__)) && \ __HIP_DEVICE_COMPILE__ @@ -51,16 +53,16 @@ #endif #if defined(__HIPCC_RTC__) - #if HIP_FP8_TYPE_FNUZ - #define ENABLE_FNUZ_HIPRTC 1 - #else - #define ENABLE_FNUZ_HIPRTC 0 - #endif - #if HIP_FP8_TYPE_OCP - #define ENABLE_OCP_HIPRTC 1 - #else - #define ENABLE_OCP_HIPRTC 0 - #endif +#if HIP_FP8_TYPE_FNUZ +#define ENABLE_FNUZ_HIPRTC 1 +#else +#define ENABLE_FNUZ_HIPRTC 0 +#endif +#if HIP_FP8_TYPE_OCP +#define ENABLE_OCP_HIPRTC 1 +#else +#define ENABLE_OCP_HIPRTC 0 +#endif #endif // Include it explicitly for HIPRTC @@ -68,6 +70,7 @@ #if !defined(__HIPCC_RTC__) #include +#include #include #include "host_defines.h" // __hip_internal:: @@ -411,7 +414,7 @@ __FP8_HOST_DEVICE_STATIC__ T cast_from_f8(__hip_fp8_storage_t x, int wm, int we, constexpr bool is_half = std::is_same::value; constexpr bool is_float = std::is_same::value; constexpr bool is_double = std::is_same::value; -#endif // defined(__clang__) and defined(__HIP__) +#endif // defined(__clang__) and defined(__HIP__) static_assert(is_half || is_float || is_double, "only half, float and double are supported"); constexpr int weo = is_half ? 5 : (is_float ? 8 : 11); @@ -482,7 +485,7 @@ __FP8_HOST_DEVICE_STATIC__ T cast_from_f8(__hip_fp8_storage_t x, int wm, int we, return fNaN; } } else if ((x & 0x7C) == 0x7C) { // e5m2 NaN/Inf - if ((x & 0x3) == 0) { // Inf + if ((x & 0x3) == 0) { // Inf if (clip) { return sign ? fmin : fmax; } @@ -1025,6 +1028,105 @@ struct __hip_fp8_e4m3_fnuz { __default_interpret)) { } +#if HIP_FP8_TYPE_FNUZ + __FP8_HOST_DEVICE__ __hip_fp8_e4m3_fnuz(long long val) { +#else + __FP8_HOST__ __hip_fp8_e4m3_fnuz(long long val) { +#endif + if (val == 0) { + __x = 0; + return; + } + + uint8_t sign = static_cast((val & 0x8000000000000000ULL) >> 56); + uint64_t absVal = (sign) ? (~val + 1) : val; + +#if __HIP_DEVICE_COMPILE__ + int msb_pos = 64 - __clzll(absVal); +#else + int msb_pos = 64 - __builtin_clzll(absVal); +#endif + + int exp = msb_pos + 7; + + if (exp >= 15) { + __x = 0x7E; + return; + } + + uint64_t mantissa_bits = 0; + if (msb_pos >= 3) { + mantissa_bits = (absVal >> (msb_pos - 3)) & 0x07; + if (msb_pos > 3) { + uint64_t remainder = absVal & ((1ULL << (msb_pos - 3)) - 1); + uint64_t halfway = 1ULL << (msb_pos - 4); + if (remainder > halfway || (remainder == halfway && (mantissa_bits & 1))) mantissa_bits++; + } + } else if (msb_pos == 2) { + mantissa_bits = (absVal & 0x03) << 1; + } else if (msb_pos == 1) { + mantissa_bits = (absVal & 0x01) << 2; + } + + if (mantissa_bits >= 8) { + mantissa_bits = 0; + exp++; + if (exp >= 15) { + __x = 0x7E; + return; + } + } + __x = sign | (exp << 3) | (mantissa_bits & 0x07); + } + +#if HIP_FP8_TYPE_FNUZ + __FP8_HOST_DEVICE__ __hip_fp8_e4m3_fnuz(unsigned long long val) { +#else + __FP8_HOST__ __hip_fp8_e4m3_fnuz(unsigned long long val) { +#endif + if (val == 0) { + __x = 0; + return; + } + +#if __HIP_DEVICE_COMPILE__ + int msb_pos = 64 - __clzll(absVal); +#else + int msb_pos = 64 - __builtin_clzll(val); +#endif + + int exp = msb_pos + 7; + + if (exp >= 15) { + __x = 0xFE; + return; + } + + uint64_t mantissa_bits = 0; + if (msb_pos >= 3) { + mantissa_bits = (val >> (msb_pos - 3)) & 0x07; + if (msb_pos > 3) { + uint64_t remainder = val & ((1ULL << (msb_pos - 3)) - 1); + uint64_t halfway = 1ULL << (msb_pos - 4); + if (remainder > halfway || (remainder == halfway && (mantissa_bits & 1))) mantissa_bits++; + } + } else if (msb_pos == 2) { + mantissa_bits = (val & 0x03) << 1; + } else if (msb_pos == 1) { + mantissa_bits = (val & 0x01) << 2; + } + + if (mantissa_bits >= 8) { + mantissa_bits = 0; + exp++; + if (exp >= 15) { + __x = 0xFE; + return; + } + } + __x = (exp << 3) | (mantissa_bits & 0x07); + } + /*! create fp8 e4m3 from double */ #if HIP_FP8_TYPE_FNUZ __FP8_HOST_DEVICE__ __hip_fp8_e4m3_fnuz(const double f) @@ -1305,16 +1407,17 @@ struct __hip_fp8_e4m3_fnuz { #endif if (internal::hip_fp8_fnuz_is_nan(__x)) { return 0; - } +} - float fval = *this; - auto llval = static_cast(fval); - if (llval <= 0) { - return 0; - } - return static_cast(fval); - } -}; +float fval = *this; +auto llval = static_cast(fval); +if (llval <= 0) { + return 0; +} +return static_cast(fval); +} +} +; /** * \brief struct representing two fp8 numbers with e4m3 interpretation @@ -1393,8 +1496,9 @@ struct __hip_fp8x2_e4m3_fnuz { internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x >> 8), __wm, __we)); #endif - } -}; +} +} +; /** * \brief struct representing four fp8 numbers with e4m3 interpretation @@ -1488,12 +1592,12 @@ struct __hip_fp8x4_e4m3_fnuz { #else __FP8_HOST__ operator float4() const { #endif - auto x = __x; // bypass const - auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E - auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); + auto x = __x; // bypass const + auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E + auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); #if HIP_FP8_CVT_FAST_PATH - float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); - float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); + float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); + float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); #else float2 high = float2(internal::cast_from_f8( static_cast<__hip_fp8_storage_t>((fp8x2_high << 8) >> 8), __wm, __we), @@ -1504,9 +1608,10 @@ struct __hip_fp8x4_e4m3_fnuz { internal::cast_from_f8( static_cast<__hip_fp8_storage_t>(fp8x2_low >> 8), __wm, __we)); #endif - return float4(low.x, low.y, high.x, high.y); - } -}; + return float4(low.x, low.y, high.x, high.y); +} +} +; /** * \brief struct representing one fp8 number with e5m2 interpretation @@ -1660,6 +1765,101 @@ struct __hip_fp8_e5m2_fnuz { return __hip_bfloat16(f); } +#if HIP_FP8_TYPE_FNUZ + __FP8_HOST_DEVICE__ __hip_fp8_e5m2_fnuz(long long val) { +#else + __FP8_HOST__ __hip_fp8_e5m2_fnuz(long long val) { +#endif + if (val == 0) { + __x = 0; + return; + } + + uint8_t sign = static_cast((val & 0x8000000000000000ULL) >> 56); + uint64_t absVal = (sign) ? (~val + 1) : val; + +#if __HIP_DEVICE_COMPILE__ + int msb_pos = 64 - __clzll(absVal); +#else + int msb_pos = 64 - __builtin_clzll(absVal); +#endif + + int exp = msb_pos + 15; + + if (exp >= 31) { + __x = 0x7E; + return; + } + + uint64_t mantissa_bits = 0; + if (msb_pos >= 2) { + mantissa_bits = (absVal >> (msb_pos - 2)) & 0x03; + if (msb_pos > 2) { + uint64_t remainder = absVal & ((1ULL << (msb_pos - 2)) - 1); + uint64_t halfway = 1ULL << (msb_pos - 3); + if (remainder > halfway || (remainder == halfway && (mantissa_bits & 1))) mantissa_bits++; + } + } else if (msb_pos == 1) { + mantissa_bits = (absVal & 1) << 1; + } + + if (mantissa_bits >= 4) { + mantissa_bits = 0; + exp++; + if (exp >= 31) { + __x = 0x7E; + return; + } + } + __x = sign | (exp << 2) | (mantissa_bits & 0x03); + } + +#if HIP_FP8_TYPE_FNUZ + __FP8_HOST_DEVICE__ __hip_fp8_e5m2_fnuz(unsigned long long val) { +#else + __FP8_HOST__ __hip_fp8_e5m2_fnuz(unsigned long long val) { +#endif + if (val == 0) { + __x = 0; + return; + } + +#if __HIP_DEVICE_COMPILE__ + int msb_pos = 64 - __clzll(absVal); +#else + int msb_pos = 64 - __builtin_clzll(val); +#endif + + int exp = msb_pos + 15; + + if (exp >= 31) { + __x = 0xFE; + return; + } + + uint64_t mantissa_bits = 0; + if (msb_pos >= 2) { + mantissa_bits = (val >> (msb_pos - 2)) & 0x03; + if (msb_pos > 2) { + uint64_t remainder = val & ((1ULL << (msb_pos - 2)) - 1); + uint64_t halfway = 1ULL << (msb_pos - 3); + if (remainder > halfway || (remainder == halfway && (mantissa_bits & 1))) mantissa_bits++; + } + } else if (msb_pos == 1) { + mantissa_bits = (val & 1) << 1; + } + + if (mantissa_bits >= 4) { + mantissa_bits = 0; + exp++; + if (exp >= 31) { + __x = 0xFE; + return; + } + } + __x = (exp << 2) | (mantissa_bits & 0x03); + } + /*! convert fp8 e4m3 to bool, return false if value is 0, true otherwise */ #if HIP_FP8_TYPE_FNUZ __FP8_HOST_DEVICE__ operator bool() const { @@ -1861,18 +2061,19 @@ struct __hip_fp8_e5m2_fnuz { #else __FP8_HOST__ operator unsigned short int() const { #endif - if (internal::hip_fp8_fnuz_is_nan(__x)) { - return 0; - } + if (internal::hip_fp8_fnuz_is_nan(__x)) { + return 0; +} - float fval = *this; - auto llval = static_cast(fval); - if (llval <= 0) { - return 0; - } - return static_cast(fval); - } -}; +float fval = *this; +auto llval = static_cast(fval); +if (llval <= 0) { + return 0; +} +return static_cast(fval); +} +} +; /** * \brief struct representing two fp8 numbers with e5m2 interpretation @@ -1944,15 +2145,16 @@ struct __hip_fp8x2_e5m2_fnuz { __FP8_HOST__ operator float2() const { #endif #if HIP_FP8_CVT_FAST_PATH - return internal::cast_to_f32x2_from_f8x2(__x, __default_interpret); + return internal::cast_to_f32x2_from_f8x2(__x, __default_interpret); #else return float2(internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x & 0xFF), __wm, __we), internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x >> 8), __wm, __we)); #endif - } -}; +} +} +; /** * \brief struct representing four fp8 numbers with e5m2 interpretation @@ -2046,12 +2248,12 @@ struct __hip_fp8x4_e5m2_fnuz { #else __FP8_HOST__ operator float4() const { #endif - auto x = __x; // bypass const - auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E - auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); + auto x = __x; // bypass const + auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E + auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); #if HIP_FP8_CVT_FAST_PATH - float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); - float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); + float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); + float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); #else float2 high = float2(internal::cast_from_f8( static_cast<__hip_fp8_storage_t>((fp8x2_high << 8) >> 8), __wm, __we), @@ -2062,11 +2264,12 @@ struct __hip_fp8x4_e5m2_fnuz { internal::cast_from_f8( static_cast<__hip_fp8_storage_t>(fp8x2_low >> 8), __wm, __we)); #endif - return float4(low.x, low.y, high.x, high.y); - } -}; + return float4(low.x, low.y, high.x, high.y); +} +} +; -#endif // ENABLE_FNUZ_HIPRTC +#endif // ENABLE_FNUZ_HIPRTC /** * \brief struct representing ocp fp8 numbers with e4m3 interpretation @@ -2419,18 +2622,19 @@ struct __hip_fp8_e4m3 { #else __FP8_HOST__ operator unsigned short int() const { #endif - if (internal::hip_fp8_ocp_is_nan(__x, __default_interpret)) { - return 0; - } + if (internal::hip_fp8_ocp_is_nan(__x, __default_interpret)) { + return 0; +} - float fval = *this; - auto llval = static_cast(fval); - if (llval <= 0) { - return 0; - } - return static_cast(fval); - } -}; +float fval = *this; +auto llval = static_cast(fval); +if (llval <= 0) { + return 0; +} +return static_cast(fval); +} +} +; /** * \brief struct representing two ocp fp8 numbers with e4m3 interpretation @@ -2503,15 +2707,16 @@ struct __hip_fp8x2_e4m3 { __FP8_HOST__ operator float2() const { #endif #if HIP_FP8_CVT_FAST_PATH - return internal::cast_to_f32x2_from_f8x2(__x, __default_interpret); + return internal::cast_to_f32x2_from_f8x2(__x, __default_interpret); #else - return float2(internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x & 0xFF), + return float2(internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x & 0xFF), __wm, __we), internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x >> 8), __wm, __we)); #endif - } -}; +} +} +; /** * \brief struct representing four ocp fp8 numbers with e4m3 interpretation @@ -2531,7 +2736,7 @@ struct __hip_fp8x4_e4m3 { #else __FP8_HOST__ __hip_fp8x4_e4m3(const double4 val) #endif - : __x{reinterpret_cast<__hip_fp8x4_storage_t>( + : __x{reinterpret_cast<__hip_fp8x4_storage_t>( static_cast(reinterpret_cast(__hip_cvt_double_to_fp8( val.x, __default_saturation, __default_interpret)) | reinterpret_cast(__hip_cvt_double_to_fp8( @@ -2606,12 +2811,12 @@ struct __hip_fp8x4_e4m3 { #else __FP8_HOST__ operator float4() const { #endif - auto x = __x; // bypass const - auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E - auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); + auto x = __x; // bypass const + auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E + auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); #if HIP_FP8_CVT_FAST_PATH - float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); - float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); + float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); + float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); #else float2 high = float2(internal::cast_from_f8( static_cast<__hip_fp8_storage_t>((fp8x2_high << 8) >> 8), __wm, __we), @@ -2622,9 +2827,10 @@ struct __hip_fp8x4_e4m3 { internal::cast_from_f8( static_cast<__hip_fp8_storage_t>(fp8x2_low >> 8), __wm, __we)); #endif - return float4(low.x, low.y, high.x, high.y); - } -}; + return float4(low.x, low.y, high.x, high.y); +} +} +; /** * \brief struct representing ocp fp8 numbers with e5m2 interpretation @@ -2981,18 +3187,19 @@ struct __hip_fp8_e5m2 { #else __FP8_HOST__ operator unsigned short int() const { #endif - if (internal::hip_fp8_ocp_is_nan(__x, __default_interpret)) { - return 0; + if (internal::hip_fp8_ocp_is_nan(__x, __default_interpret)) { + return 0; } - float fval = *this; - auto llval = static_cast(fval); - if (llval <= 0) { - return 0; - } - return static_cast(fval); - } -}; +float fval = *this; +auto llval = static_cast(fval); +if (llval <= 0) { + return 0; +} +return static_cast(fval); +} +} +; /** * \brief struct representing two ocp fp8 numbers with e5m2 interpretation @@ -3065,16 +3272,17 @@ struct __hip_fp8x2_e5m2 { __FP8_HOST__ operator float2() const { #endif #if HIP_FP8_CVT_FAST_PATH - return internal::cast_to_f32x2_from_f8x2(__x, __default_interpret); + return internal::cast_to_f32x2_from_f8x2(__x, __default_interpret); #else return float2( - internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x & 0xFF), __wm, + internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x & 0xFF), __wm, __we, __default_saturation == __HIP_SATFINITE), - internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x >> 8), __wm, __we, + internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x >> 8), __wm, __we, __default_saturation == __HIP_SATFINITE)); #endif - } -}; +} +} +; /** * \brief struct representing four ocp fp8 numbers with e5m2 interpretation @@ -3168,12 +3376,12 @@ struct __hip_fp8x4_e5m2 { #else __FP8_HOST__ operator float4() const { #endif - auto x = __x; // bypass const - auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E - auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); + auto x = __x; // bypass const + auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E + auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); #if HIP_FP8_CVT_FAST_PATH - float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); - float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); + float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); + float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); #else float2 high = float2( internal::cast_from_f8( @@ -3188,8 +3396,9 @@ struct __hip_fp8x4_e5m2 { internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(fp8x2_low >> 8), __wm, __we, __default_saturation == __HIP_SATFINITE)); #endif - return float4(low.x, low.y, high.x, high.y); - } -}; -#endif // ENABLE_OCP_HIPRTC -#endif // _HIP_INCLUDE_HIP_AMD_DETAIL_HIP_FP8_H_ + return float4(low.x, low.y, high.x, high.y); +} +} +; +#endif // ENABLE_OCP_HIPRTC +#endif // _HIP_INCLUDE_HIP_AMD_DETAIL_HIP_FP8_H_