From 85ea1082f6191751fea727f74daa0393b32ee0ef Mon Sep 17 00:00:00 2001 From: Danylo Lytovchenko Date: Tue, 8 Jul 2025 09:06:56 +0200 Subject: [PATCH 1/2] SWDEV-452411 - add fp8 conversions from long long --- hipamd/include/hip/amd_detail/amd_hip_fp8.h | 466 +++++++++++++++----- 1 file changed, 363 insertions(+), 103 deletions(-) diff --git a/hipamd/include/hip/amd_detail/amd_hip_fp8.h b/hipamd/include/hip/amd_detail/amd_hip_fp8.h index 18823f16b8a2..6a35a055b1ee 100644 --- a/hipamd/include/hip/amd_detail/amd_hip_fp8.h +++ b/hipamd/include/hip/amd_detail/amd_hip_fp8.h @@ -30,6 +30,8 @@ #ifndef _HIP_INCLUDE_HIP_AMD_DETAIL_HIP_FP8_H_ #define _HIP_INCLUDE_HIP_AMD_DETAIL_HIP_FP8_H_ +#include +#include "device_library_decls.h" #if (defined(__gfx942__) || defined(__gfx1200__) || defined(__gfx1201__) || \ defined(__gfx950__)) && \ __HIP_DEVICE_COMPILE__ @@ -51,16 +53,16 @@ #endif #if defined(__HIPCC_RTC__) - #if HIP_FP8_TYPE_FNUZ - #define ENABLE_FNUZ_HIPRTC 1 - #else - #define ENABLE_FNUZ_HIPRTC 0 - #endif - #if HIP_FP8_TYPE_OCP - #define ENABLE_OCP_HIPRTC 1 - #else - #define ENABLE_OCP_HIPRTC 0 - #endif +#if HIP_FP8_TYPE_FNUZ +#define ENABLE_FNUZ_HIPRTC 1 +#else +#define ENABLE_FNUZ_HIPRTC 0 +#endif +#if HIP_FP8_TYPE_OCP +#define ENABLE_OCP_HIPRTC 1 +#else +#define ENABLE_OCP_HIPRTC 0 +#endif #endif // Include it explicitly for HIPRTC @@ -411,7 +413,7 @@ __FP8_HOST_DEVICE_STATIC__ T cast_from_f8(__hip_fp8_storage_t x, int wm, int we, constexpr bool is_half = std::is_same::value; constexpr bool is_float = std::is_same::value; constexpr bool is_double = std::is_same::value; -#endif // defined(__clang__) and defined(__HIP__) +#endif // defined(__clang__) and defined(__HIP__) static_assert(is_half || is_float || is_double, "only half, float and double are supported"); constexpr int weo = is_half ? 5 : (is_float ? 8 : 11); @@ -482,7 +484,7 @@ __FP8_HOST_DEVICE_STATIC__ T cast_from_f8(__hip_fp8_storage_t x, int wm, int we, return fNaN; } } else if ((x & 0x7C) == 0x7C) { // e5m2 NaN/Inf - if ((x & 0x3) == 0) { // Inf + if ((x & 0x3) == 0) { // Inf if (clip) { return sign ? fmin : fmax; } @@ -1025,6 +1027,131 @@ struct __hip_fp8_e4m3_fnuz { __default_interpret)) { } +#if HIP_FP8_TYPE_FNUZ + __FP8_HOST_DEVICE__ __hip_fp8_e4m3_fnuz(long long val) { +#else + __FP8_HOST__ __hip_fp8_e4m3_fnuz(long long val) { +#endif + if (val == 0) { + __x = 0; + return; + } + + uint8_t sign = (val < 0) ? 1 : 0; + uint64_t absVal = (val < 0) ? -val : val; + + int msb_pos = 63; + while (msb_pos >= 0 && !(absVal & (1ULL << msb_pos))) msb_pos--; + + int exp = msb_pos + 7; + + if (exp <= 0) { + int shift = -6 - msb_pos; + uint64_t shifted = absVal >> shift; + + uint64_t mantissa = shifted; + if (shift > 0) { + uint64_t remainder = absVal & ((1ULL << shift) - 1); + uint64_t halfway = 1ULL << (shift - 1); + if (remainder > halfway || (remainder == halfway && (mantissa & 1))) mantissa++; + } + + if (mantissa >= 8) mantissa = 7; + __x = (sign << 7) | (mantissa & 0x07); + return; + } + + if (exp >= 15) { + __x = (sign << 7) | (0b1110 << 3) | 7; + return; + } + + uint64_t mantissa_bits = 0; + if (msb_pos >= 3) { + mantissa_bits = (absVal >> (msb_pos - 3)) & 0x07; + if (msb_pos > 3) { + uint64_t remainder = absVal & ((1ULL << (msb_pos - 3)) - 1); + uint64_t halfway = 1ULL << (msb_pos - 4); + if (remainder > halfway || (remainder == halfway && (mantissa_bits & 1))) mantissa_bits++; + } + } else if (msb_pos == 2) { + mantissa_bits = (absVal & 0x03) << 1; + } else if (msb_pos == 1) { + mantissa_bits = (absVal & 0x01) << 2; + } + + if (mantissa_bits >= 8) { + mantissa_bits = 0; + exp++; + if (exp >= 15) { + __x = (sign << 7) | (0b1110 << 3) | 3; + return; + } + } + __x = (sign << 7) | (exp << 3) | (mantissa_bits & 0x07); + } + +#if HIP_FP8_TYPE_FNUZ + __FP8_HOST_DEVICE__ __hip_fp8_e4m3_fnuz(unsigned long long val) { +#else + __FP8_HOST__ __hip_fp8_e4m3_fnuz(unsigned long long val) { +#endif + if (val == 0) { + __x = 0; + return; + } + + int msb_pos = 63; + while (msb_pos >= 0 && !(val & (1ULL << msb_pos))) msb_pos--; + + int exp = msb_pos + 7; + + if (exp <= 0) { + int shift = -6 - msb_pos; + uint64_t shifted = val >> shift; + + uint64_t mantissa = shifted; + if (shift > 0) { + uint64_t remainder = val & ((1ULL << shift) - 1); + uint64_t halfway = 1ULL << (shift - 1); + if (remainder > halfway || (remainder == halfway && (mantissa & 1))) mantissa++; + } + + if (mantissa >= 8) mantissa = 7; + __x = (mantissa & 0x07); + return; + } + + if (exp >= 15) { + __x = (0b1110 << 3) | 7; + return; + } + + uint64_t mantissa_bits = 0; + if (msb_pos >= 3) { + mantissa_bits = (val >> (msb_pos - 3)) & 0x07; + if (msb_pos > 3) { + uint64_t remainder = val & ((1ULL << (msb_pos - 3)) - 1); + uint64_t halfway = 1ULL << (msb_pos - 4); + if (remainder > halfway || (remainder == halfway && (mantissa_bits & 1))) mantissa_bits++; + } + } else if (msb_pos == 2) { + mantissa_bits = (val & 0x03) << 1; + } else if (msb_pos == 1) { + mantissa_bits = (val & 0x01) << 2; + } + + if (mantissa_bits >= 8) { + mantissa_bits = 0; + exp++; + if (exp >= 15) { + __x = (0b1110 << 3) | 7; + return; + } + } + __x = (exp << 3) | (mantissa_bits & 0x07); + } + /*! create fp8 e4m3 from double */ #if HIP_FP8_TYPE_FNUZ __FP8_HOST_DEVICE__ __hip_fp8_e4m3_fnuz(const double f) @@ -1305,16 +1432,17 @@ struct __hip_fp8_e4m3_fnuz { #endif if (internal::hip_fp8_fnuz_is_nan(__x)) { return 0; - } +} - float fval = *this; - auto llval = static_cast(fval); - if (llval <= 0) { - return 0; - } - return static_cast(fval); - } -}; +float fval = *this; +auto llval = static_cast(fval); +if (llval <= 0) { + return 0; +} +return static_cast(fval); +} +} +; /** * \brief struct representing two fp8 numbers with e4m3 interpretation @@ -1393,8 +1521,9 @@ struct __hip_fp8x2_e4m3_fnuz { internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x >> 8), __wm, __we)); #endif - } -}; +} +} +; /** * \brief struct representing four fp8 numbers with e4m3 interpretation @@ -1488,12 +1617,12 @@ struct __hip_fp8x4_e4m3_fnuz { #else __FP8_HOST__ operator float4() const { #endif - auto x = __x; // bypass const - auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E - auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); + auto x = __x; // bypass const + auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E + auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); #if HIP_FP8_CVT_FAST_PATH - float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); - float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); + float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); + float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); #else float2 high = float2(internal::cast_from_f8( static_cast<__hip_fp8_storage_t>((fp8x2_high << 8) >> 8), __wm, __we), @@ -1504,9 +1633,10 @@ struct __hip_fp8x4_e4m3_fnuz { internal::cast_from_f8( static_cast<__hip_fp8_storage_t>(fp8x2_low >> 8), __wm, __we)); #endif - return float4(low.x, low.y, high.x, high.y); - } -}; + return float4(low.x, low.y, high.x, high.y); +} +} +; /** * \brief struct representing one fp8 number with e5m2 interpretation @@ -1660,6 +1790,127 @@ struct __hip_fp8_e5m2_fnuz { return __hip_bfloat16(f); } +#if HIP_FP8_TYPE_FNUZ + __FP8_HOST_DEVICE__ __hip_fp8_e5m2_fnuz(long long val) { +#else + __FP8_HOST__ __hip_fp8_e5m2_fnuz(long long val) { +#endif + if (val == 0) { + __x = 0; + return; + } + + uint8_t sign = (val < 0) ? 1 : 0; + uint64_t absVal = (val < 0) ? -val : val; + + int msb_pos = 63; + while (msb_pos >= 0 && !(absVal & (1ULL << msb_pos))) msb_pos--; + + int exp = msb_pos + 15; + + if (exp <= 0) { + int shift = -14 - msb_pos; + uint64_t shifted = absVal >> shift; + + uint64_t mantissa = shifted; + if (shift > 0) { + uint64_t remainder = absVal & ((1ULL << shift) - 1); + uint64_t halfway = 1ULL << (shift - 1); + if (remainder > halfway || (remainder == halfway && (mantissa & 1))) mantissa++; + } + + if (mantissa >= 4) mantissa = 3; + __x = (sign << 7) | (mantissa & 0x03); + return; + } + + if (exp >= 31) { + __x = (sign << 7) | (0b11110 << 2) | 3; + return; + } + + uint64_t mantissa_bits = 0; + if (msb_pos >= 2) { + mantissa_bits = (absVal >> (msb_pos - 2)) & 0x03; + if (msb_pos > 2) { + uint64_t remainder = absVal & ((1ULL << (msb_pos - 2)) - 1); + uint64_t halfway = 1ULL << (msb_pos - 3); + if (remainder > halfway || (remainder == halfway && (mantissa_bits & 1))) mantissa_bits++; + } + } else if (msb_pos == 1) { + mantissa_bits = (absVal & 1) << 1; + } + + if (mantissa_bits >= 4) { + mantissa_bits = 0; + exp++; + if (exp >= 31) { + __x = (sign << 7) | (0b11110 << 2) | 3; + return; + } + } + __x = (sign << 7) | (exp << 2) | (mantissa_bits & 0x03); + } + +#if HIP_FP8_TYPE_FNUZ + __FP8_HOST_DEVICE__ __hip_fp8_e5m2_fnuz(unsigned long long val) { +#else + __FP8_HOST__ __hip_fp8_e5m2_fnuz(unsigned long long val) { +#endif + if (val == 0) { + __x = 0; + return; + } + + int msb_pos = 63; + while (msb_pos >= 0 && !(val & (1ULL << msb_pos))) msb_pos--; + + int exp = msb_pos + 15; + + if (exp <= 0) { + int shift = -14 - msb_pos; + uint64_t shifted = val >> shift; + + uint64_t mantissa = shifted; + if (shift > 0) { + uint64_t remainder = val & ((1ULL << shift) - 1); + uint64_t halfway = 1ULL << (shift - 1); + if (remainder > halfway || (remainder == halfway && (mantissa & 1))) mantissa++; + } + + if (mantissa >= 4) mantissa = 3; + __x = (mantissa & 0x03); + return; + } + + if (exp >= 31) { + __x = (0b11110 << 2) | 3; + return; + } + + uint64_t mantissa_bits = 0; + if (msb_pos >= 2) { + mantissa_bits = (val >> (msb_pos - 2)) & 0x03; + if (msb_pos > 2) { + uint64_t remainder = val & ((1ULL << (msb_pos - 2)) - 1); + uint64_t halfway = 1ULL << (msb_pos - 3); + if (remainder > halfway || (remainder == halfway && (mantissa_bits & 1))) mantissa_bits++; + } + } else if (msb_pos == 1) { + mantissa_bits = (val & 1) << 1; + } + + if (mantissa_bits >= 4) { + mantissa_bits = 0; + exp++; + if (exp >= 31) { + __x = (0b11110 << 2) | 3; + return; + } + } + __x = (exp << 2) | (mantissa_bits & 0x03); + } + /*! convert fp8 e4m3 to bool, return false if value is 0, true otherwise */ #if HIP_FP8_TYPE_FNUZ __FP8_HOST_DEVICE__ operator bool() const { @@ -1861,18 +2112,19 @@ struct __hip_fp8_e5m2_fnuz { #else __FP8_HOST__ operator unsigned short int() const { #endif - if (internal::hip_fp8_fnuz_is_nan(__x)) { - return 0; - } + if (internal::hip_fp8_fnuz_is_nan(__x)) { + return 0; +} - float fval = *this; - auto llval = static_cast(fval); - if (llval <= 0) { - return 0; - } - return static_cast(fval); - } -}; +float fval = *this; +auto llval = static_cast(fval); +if (llval <= 0) { + return 0; +} +return static_cast(fval); +} +} +; /** * \brief struct representing two fp8 numbers with e5m2 interpretation @@ -1944,15 +2196,16 @@ struct __hip_fp8x2_e5m2_fnuz { __FP8_HOST__ operator float2() const { #endif #if HIP_FP8_CVT_FAST_PATH - return internal::cast_to_f32x2_from_f8x2(__x, __default_interpret); + return internal::cast_to_f32x2_from_f8x2(__x, __default_interpret); #else return float2(internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x & 0xFF), __wm, __we), internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x >> 8), __wm, __we)); #endif - } -}; +} +} +; /** * \brief struct representing four fp8 numbers with e5m2 interpretation @@ -2046,12 +2299,12 @@ struct __hip_fp8x4_e5m2_fnuz { #else __FP8_HOST__ operator float4() const { #endif - auto x = __x; // bypass const - auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E - auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); + auto x = __x; // bypass const + auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E + auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); #if HIP_FP8_CVT_FAST_PATH - float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); - float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); + float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); + float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); #else float2 high = float2(internal::cast_from_f8( static_cast<__hip_fp8_storage_t>((fp8x2_high << 8) >> 8), __wm, __we), @@ -2062,11 +2315,12 @@ struct __hip_fp8x4_e5m2_fnuz { internal::cast_from_f8( static_cast<__hip_fp8_storage_t>(fp8x2_low >> 8), __wm, __we)); #endif - return float4(low.x, low.y, high.x, high.y); - } -}; + return float4(low.x, low.y, high.x, high.y); +} +} +; -#endif // ENABLE_FNUZ_HIPRTC +#endif // ENABLE_FNUZ_HIPRTC /** * \brief struct representing ocp fp8 numbers with e4m3 interpretation @@ -2419,18 +2673,19 @@ struct __hip_fp8_e4m3 { #else __FP8_HOST__ operator unsigned short int() const { #endif - if (internal::hip_fp8_ocp_is_nan(__x, __default_interpret)) { - return 0; - } + if (internal::hip_fp8_ocp_is_nan(__x, __default_interpret)) { + return 0; +} - float fval = *this; - auto llval = static_cast(fval); - if (llval <= 0) { - return 0; - } - return static_cast(fval); - } -}; +float fval = *this; +auto llval = static_cast(fval); +if (llval <= 0) { + return 0; +} +return static_cast(fval); +} +} +; /** * \brief struct representing two ocp fp8 numbers with e4m3 interpretation @@ -2503,15 +2758,16 @@ struct __hip_fp8x2_e4m3 { __FP8_HOST__ operator float2() const { #endif #if HIP_FP8_CVT_FAST_PATH - return internal::cast_to_f32x2_from_f8x2(__x, __default_interpret); + return internal::cast_to_f32x2_from_f8x2(__x, __default_interpret); #else - return float2(internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x & 0xFF), + return float2(internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x & 0xFF), __wm, __we), internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x >> 8), __wm, __we)); #endif - } -}; +} +} +; /** * \brief struct representing four ocp fp8 numbers with e4m3 interpretation @@ -2531,7 +2787,7 @@ struct __hip_fp8x4_e4m3 { #else __FP8_HOST__ __hip_fp8x4_e4m3(const double4 val) #endif - : __x{reinterpret_cast<__hip_fp8x4_storage_t>( + : __x{reinterpret_cast<__hip_fp8x4_storage_t>( static_cast(reinterpret_cast(__hip_cvt_double_to_fp8( val.x, __default_saturation, __default_interpret)) | reinterpret_cast(__hip_cvt_double_to_fp8( @@ -2606,12 +2862,12 @@ struct __hip_fp8x4_e4m3 { #else __FP8_HOST__ operator float4() const { #endif - auto x = __x; // bypass const - auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E - auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); + auto x = __x; // bypass const + auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E + auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); #if HIP_FP8_CVT_FAST_PATH - float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); - float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); + float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); + float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); #else float2 high = float2(internal::cast_from_f8( static_cast<__hip_fp8_storage_t>((fp8x2_high << 8) >> 8), __wm, __we), @@ -2622,9 +2878,10 @@ struct __hip_fp8x4_e4m3 { internal::cast_from_f8( static_cast<__hip_fp8_storage_t>(fp8x2_low >> 8), __wm, __we)); #endif - return float4(low.x, low.y, high.x, high.y); - } -}; + return float4(low.x, low.y, high.x, high.y); +} +} +; /** * \brief struct representing ocp fp8 numbers with e5m2 interpretation @@ -2981,18 +3238,19 @@ struct __hip_fp8_e5m2 { #else __FP8_HOST__ operator unsigned short int() const { #endif - if (internal::hip_fp8_ocp_is_nan(__x, __default_interpret)) { - return 0; + if (internal::hip_fp8_ocp_is_nan(__x, __default_interpret)) { + return 0; } - float fval = *this; - auto llval = static_cast(fval); - if (llval <= 0) { - return 0; - } - return static_cast(fval); - } -}; +float fval = *this; +auto llval = static_cast(fval); +if (llval <= 0) { + return 0; +} +return static_cast(fval); +} +} +; /** * \brief struct representing two ocp fp8 numbers with e5m2 interpretation @@ -3065,16 +3323,17 @@ struct __hip_fp8x2_e5m2 { __FP8_HOST__ operator float2() const { #endif #if HIP_FP8_CVT_FAST_PATH - return internal::cast_to_f32x2_from_f8x2(__x, __default_interpret); + return internal::cast_to_f32x2_from_f8x2(__x, __default_interpret); #else return float2( - internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x & 0xFF), __wm, + internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x & 0xFF), __wm, __we, __default_saturation == __HIP_SATFINITE), - internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x >> 8), __wm, __we, + internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(__x >> 8), __wm, __we, __default_saturation == __HIP_SATFINITE)); #endif - } -}; +} +} +; /** * \brief struct representing four ocp fp8 numbers with e5m2 interpretation @@ -3168,12 +3427,12 @@ struct __hip_fp8x4_e5m2 { #else __FP8_HOST__ operator float4() const { #endif - auto x = __x; // bypass const - auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E - auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); + auto x = __x; // bypass const + auto fp8x2_low = *reinterpret_cast<__hip_fp8x2_storage_t*>(&x); // Little E + auto fp8x2_high = *(reinterpret_cast<__hip_fp8x2_storage_t*>(&x) + 1); #if HIP_FP8_CVT_FAST_PATH - float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); - float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); + float2 high = internal::cast_to_f32x2_from_f8x2(fp8x2_high, __default_interpret); + float2 low = internal::cast_to_f32x2_from_f8x2(fp8x2_low, __default_interpret); #else float2 high = float2( internal::cast_from_f8( @@ -3188,8 +3447,9 @@ struct __hip_fp8x4_e5m2 { internal::cast_from_f8(static_cast<__hip_fp8_storage_t>(fp8x2_low >> 8), __wm, __we, __default_saturation == __HIP_SATFINITE)); #endif - return float4(low.x, low.y, high.x, high.y); - } -}; -#endif // ENABLE_OCP_HIPRTC -#endif // _HIP_INCLUDE_HIP_AMD_DETAIL_HIP_FP8_H_ + return float4(low.x, low.y, high.x, high.y); +} +} +; +#endif // ENABLE_OCP_HIPRTC +#endif // _HIP_INCLUDE_HIP_AMD_DETAIL_HIP_FP8_H_ From 94af21ba52261744d7b0ed35e492c2a2ba21950a Mon Sep 17 00:00:00 2001 From: Danylo Lytovchenko Date: Thu, 10 Jul 2025 16:02:55 +0200 Subject: [PATCH 2/2] SWDEV-452411 - reduce branching, use builtins for msb position, use saturation --- hipamd/include/hip/amd_detail/amd_hip_fp8.h | 121 ++++++-------------- 1 file changed, 35 insertions(+), 86 deletions(-) diff --git a/hipamd/include/hip/amd_detail/amd_hip_fp8.h b/hipamd/include/hip/amd_detail/amd_hip_fp8.h index 6a35a055b1ee..fa3e7b4dcac6 100644 --- a/hipamd/include/hip/amd_detail/amd_hip_fp8.h +++ b/hipamd/include/hip/amd_detail/amd_hip_fp8.h @@ -70,6 +70,7 @@ #if !defined(__HIPCC_RTC__) #include +#include #include #include "host_defines.h" // __hip_internal:: @@ -1037,32 +1038,19 @@ struct __hip_fp8_e4m3_fnuz { return; } - uint8_t sign = (val < 0) ? 1 : 0; - uint64_t absVal = (val < 0) ? -val : val; + uint8_t sign = static_cast((val & 0x8000000000000000ULL) >> 56); + uint64_t absVal = (sign) ? (~val + 1) : val; - int msb_pos = 63; - while (msb_pos >= 0 && !(absVal & (1ULL << msb_pos))) msb_pos--; +#if __HIP_DEVICE_COMPILE__ + int msb_pos = 64 - __clzll(absVal); +#else + int msb_pos = 64 - __builtin_clzll(absVal); +#endif int exp = msb_pos + 7; - if (exp <= 0) { - int shift = -6 - msb_pos; - uint64_t shifted = absVal >> shift; - - uint64_t mantissa = shifted; - if (shift > 0) { - uint64_t remainder = absVal & ((1ULL << shift) - 1); - uint64_t halfway = 1ULL << (shift - 1); - if (remainder > halfway || (remainder == halfway && (mantissa & 1))) mantissa++; - } - - if (mantissa >= 8) mantissa = 7; - __x = (sign << 7) | (mantissa & 0x07); - return; - } - if (exp >= 15) { - __x = (sign << 7) | (0b1110 << 3) | 7; + __x = 0x7E; return; } @@ -1084,11 +1072,11 @@ struct __hip_fp8_e4m3_fnuz { mantissa_bits = 0; exp++; if (exp >= 15) { - __x = (sign << 7) | (0b1110 << 3) | 3; + __x = 0x7E; return; } } - __x = (sign << 7) | (exp << 3) | (mantissa_bits & 0x07); + __x = sign | (exp << 3) | (mantissa_bits & 0x07); } #if HIP_FP8_TYPE_FNUZ @@ -1101,29 +1089,16 @@ struct __hip_fp8_e4m3_fnuz { return; } - int msb_pos = 63; - while (msb_pos >= 0 && !(val & (1ULL << msb_pos))) msb_pos--; +#if __HIP_DEVICE_COMPILE__ + int msb_pos = 64 - __clzll(absVal); +#else + int msb_pos = 64 - __builtin_clzll(val); +#endif int exp = msb_pos + 7; - if (exp <= 0) { - int shift = -6 - msb_pos; - uint64_t shifted = val >> shift; - - uint64_t mantissa = shifted; - if (shift > 0) { - uint64_t remainder = val & ((1ULL << shift) - 1); - uint64_t halfway = 1ULL << (shift - 1); - if (remainder > halfway || (remainder == halfway && (mantissa & 1))) mantissa++; - } - - if (mantissa >= 8) mantissa = 7; - __x = (mantissa & 0x07); - return; - } - if (exp >= 15) { - __x = (0b1110 << 3) | 7; + __x = 0xFE; return; } @@ -1145,7 +1120,7 @@ struct __hip_fp8_e4m3_fnuz { mantissa_bits = 0; exp++; if (exp >= 15) { - __x = (0b1110 << 3) | 7; + __x = 0xFE; return; } } @@ -1800,32 +1775,19 @@ struct __hip_fp8_e5m2_fnuz { return; } - uint8_t sign = (val < 0) ? 1 : 0; - uint64_t absVal = (val < 0) ? -val : val; + uint8_t sign = static_cast((val & 0x8000000000000000ULL) >> 56); + uint64_t absVal = (sign) ? (~val + 1) : val; - int msb_pos = 63; - while (msb_pos >= 0 && !(absVal & (1ULL << msb_pos))) msb_pos--; +#if __HIP_DEVICE_COMPILE__ + int msb_pos = 64 - __clzll(absVal); +#else + int msb_pos = 64 - __builtin_clzll(absVal); +#endif int exp = msb_pos + 15; - if (exp <= 0) { - int shift = -14 - msb_pos; - uint64_t shifted = absVal >> shift; - - uint64_t mantissa = shifted; - if (shift > 0) { - uint64_t remainder = absVal & ((1ULL << shift) - 1); - uint64_t halfway = 1ULL << (shift - 1); - if (remainder > halfway || (remainder == halfway && (mantissa & 1))) mantissa++; - } - - if (mantissa >= 4) mantissa = 3; - __x = (sign << 7) | (mantissa & 0x03); - return; - } - if (exp >= 31) { - __x = (sign << 7) | (0b11110 << 2) | 3; + __x = 0x7E; return; } @@ -1845,11 +1807,11 @@ struct __hip_fp8_e5m2_fnuz { mantissa_bits = 0; exp++; if (exp >= 31) { - __x = (sign << 7) | (0b11110 << 2) | 3; + __x = 0x7E; return; } } - __x = (sign << 7) | (exp << 2) | (mantissa_bits & 0x03); + __x = sign | (exp << 2) | (mantissa_bits & 0x03); } #if HIP_FP8_TYPE_FNUZ @@ -1862,29 +1824,16 @@ struct __hip_fp8_e5m2_fnuz { return; } - int msb_pos = 63; - while (msb_pos >= 0 && !(val & (1ULL << msb_pos))) msb_pos--; +#if __HIP_DEVICE_COMPILE__ + int msb_pos = 64 - __clzll(absVal); +#else + int msb_pos = 64 - __builtin_clzll(val); +#endif int exp = msb_pos + 15; - if (exp <= 0) { - int shift = -14 - msb_pos; - uint64_t shifted = val >> shift; - - uint64_t mantissa = shifted; - if (shift > 0) { - uint64_t remainder = val & ((1ULL << shift) - 1); - uint64_t halfway = 1ULL << (shift - 1); - if (remainder > halfway || (remainder == halfway && (mantissa & 1))) mantissa++; - } - - if (mantissa >= 4) mantissa = 3; - __x = (mantissa & 0x03); - return; - } - if (exp >= 31) { - __x = (0b11110 << 2) | 3; + __x = 0xFE; return; } @@ -1904,7 +1853,7 @@ struct __hip_fp8_e5m2_fnuz { mantissa_bits = 0; exp++; if (exp >= 31) { - __x = (0b11110 << 2) | 3; + __x = 0xFE; return; } }