diff --git a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp index 6d5de93ed348..a3db12bb4975 100644 --- a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp +++ b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp @@ -1881,10 +1881,16 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - _mm_storeu_si128((__m128i*)dstPtr, px128[0]); - _mm_storeu_si128((__m128i*)(dstPtr + 3), px128[1]); - _mm_storeu_si128((__m128i*)(dstPtr + 6), px128[2]); - _mm_storeu_si128((__m128i*)(dstPtr + 9), px128[3]); + __m128i packed01 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + __m128i packed23 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + rpp_storeu_si64(dstPtr + 0, packed01); + rpp_storeu_si32(dstPtr + 4, _mm_srli_si128(packed01, 8)); + rpp_storeu_si64(dstPtr + 6, packed23); + rpp_storeu_si32(dstPtr + 10, _mm_srli_si128(packed23, 8)); p128[0] = _mm256_extractf128_ps(p[0], 1); p128[1] = _mm256_extractf128_ps(p[2], 1); @@ -1894,10 +1900,16 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - _mm_storeu_si128((__m128i*)(dstPtr + 12), px128[0]); - _mm_storeu_si128((__m128i*)(dstPtr + 15), px128[1]); - _mm_storeu_si128((__m128i*)(dstPtr + 18), px128[2]); - _mm_storeu_si128((__m128i*)(dstPtr + 21), px128[3]); + packed01 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed23 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + rpp_storeu_si64(dstPtr + 12, packed01); + rpp_storeu_si32(dstPtr + 16, _mm_srli_si128(packed01, 8)); + rpp_storeu_si64(dstPtr + 18, packed23); + rpp_storeu_si32(dstPtr + 22, _mm_srli_si128(packed23, 8)); p128[0] = _mm256_extractf128_ps(p[1], 0); p128[1] = _mm256_extractf128_ps(p[3], 0); @@ -1907,10 +1919,16 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - _mm_storeu_si128((__m128i*)(dstPtr + 24), px128[0]); - _mm_storeu_si128((__m128i*)(dstPtr + 27), px128[1]); - _mm_storeu_si128((__m128i*)(dstPtr + 30), px128[2]); - _mm_storeu_si128((__m128i*)(dstPtr + 33), px128[3]); + packed01 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed23 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + rpp_storeu_si64(dstPtr + 24, packed01); + rpp_storeu_si32(dstPtr + 28, _mm_srli_si128(packed01, 8)); + rpp_storeu_si64(dstPtr + 30, packed23); + rpp_storeu_si32(dstPtr + 34, _mm_srli_si128(packed23, 8)); p128[0] = _mm256_extractf128_ps(p[1], 1); p128[1] = _mm256_extractf128_ps(p[3], 1); @@ -1920,10 +1938,16 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - _mm_storeu_si128((__m128i*)(dstPtr + 36), px128[0]); - _mm_storeu_si128((__m128i*)(dstPtr + 39), px128[1]); - _mm_storeu_si128((__m128i*)(dstPtr + 42), px128[2]); - _mm_storeu_si128((__m128i*)(dstPtr + 45), px128[3]); + packed01 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed23 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + rpp_storeu_si64(dstPtr + 36, packed01); + rpp_storeu_si32(dstPtr + 40, _mm_srli_si128(packed01, 8)); + rpp_storeu_si64(dstPtr + 42, packed23); + rpp_storeu_si32(dstPtr + 46, _mm_srli_si128(packed23, 8)); } inline void rpp_load48_u8pln3_to_f32pln3_avx(Rpp8u* srcPtrR, Rpp8u* srcPtrG, Rpp8u* srcPtrB, @@ -2083,9 +2107,11 @@ inline void rpp_store48_f32pln3_to_u8pkd3_avx(Rpp8u* dstPtr, __m256* p) { _mm_storeu_si128( (__m128i*)(dstPtr + 24), px[2]); /* store [R09|G09|B09|R10|G10|B10|R11|G11|B11|R12|G12|B12|00|00|00|00] */ - _mm_storeu_si128( - (__m128i*)(dstPtr + 36), - px[3]); /* store [R13|G13|B13|R14|G14|B14|R15|G15|B15|R16|G16|B16|00|00|00|00] */ + // Fixed: Only store 12 bytes (not 16) to avoid writing beyond 48-byte boundary + _mm_storel_epi64((__m128i*)(dstPtr + 36), + px[3]); /* store first 8 bytes [R13|G13|B13|R14|G14|B14|R15|G15] */ + rpp_storeu_si32(dstPtr + 44, + _mm_srli_si128(px[3], 8)); /* store last 4 bytes [B15|R16|G16|B16] */ } inline void rpp_load24_u8pln3_to_f64pln3_avx(Rpp8u* srcPtrR, Rpp8u* srcPtrG, Rpp8u* srcPtrB, @@ -2596,7 +2622,11 @@ inline void rpp_store24_f32pln3_to_f32pkd3_avx(Rpp32f* dstPtr, __m256* p) { _mm_storeu_ps(dstPtr + 12, p128[0]); _mm_storeu_ps(dstPtr + 15, p128[1]); _mm_storeu_ps(dstPtr + 18, p128[2]); - _mm_storeu_ps(dstPtr + 21, p128[3]); + // Fixed: Only store 3 floats (not 4) to avoid writing beyond 24-float (96-byte) boundary + rpp_storeu_si64(dstPtr + 21, _mm_castps_si128(p128[3])); /* store first 2 floats [R08|G08] */ + _mm_store_ss(dstPtr + 23, + _mm_castsi128_ps(_mm_srli_si128(_mm_castps_si128(p128[3]), + 8))); /* store 3rd float [B08] using shift */ } inline void extract_and_convert_to_f64_pln3_to_pln3(__m256* px, __m256d* p) { @@ -3053,9 +3083,11 @@ inline void rpp_store48_f32pln3_to_i8pkd3_avx(Rpp8s* dstPtr, __m256* p) { _mm_storeu_si128( (__m128i*)(dstPtr + 24), px[2]); /* store [R09|G09|B09|R10|G10|B10|R11|G11|B11|R12|G12|B12|00|00|00|00] */ - _mm_storeu_si128( - (__m128i*)(dstPtr + 36), - px[3]); /* store [R13|G13|B13|R14|G14|B14|R15|G15|B15|R16|G16|B16|00|00|00|00] */ + // Fixed: Only store 12 bytes (not 16) to avoid writing beyond 48-byte boundary + _mm_storel_epi64((__m128i*)(dstPtr + 36), + px[3]); /* store first 8 bytes [R13|G13|B13|R14|G14|B14|R15|G15] */ + rpp_storeu_si32(dstPtr + 44, + _mm_srli_si128(px[3], 8)); /* store last 4 bytes [B15|R16|G16|B16] */ } inline void rpp_load24_i8pln3_to_f64pln3_avx(Rpp8s* srcPtrR, Rpp8s* srcPtrG, Rpp8s* srcPtrB, @@ -4749,10 +4781,18 @@ inline void rpp_store24_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - _mm_storeu_si128((__m128i*)dstPtr, px128[0]); - _mm_storeu_si128((__m128i*)(dstPtr + 3), px128[1]); - _mm_storeu_si128((__m128i*)(dstPtr + 6), px128[2]); - _mm_storeu_si128((__m128i*)(dstPtr + 9), px128[3]); + // Each px128[i] contains [Ri, Gi, Bi, unused] - pack pairs without unused elements + // Shuffle to pack: px128[0]+px128[1] -> [R0,G0,B0,R1,G1,B1,X,X] + __m128i packed01 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + __m128i packed23 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + rpp_storeu_si64(dstPtr + 0, packed01); // Store R0,G0,B0,R1 + rpp_storeu_si32(dstPtr + 4, _mm_srli_si128(packed01, 8)); // Store G1,B1 + rpp_storeu_si64(dstPtr + 6, packed23); // Store R2,G2,B2,R3 + rpp_storeu_si32(dstPtr + 10, _mm_srli_si128(packed23, 8)); // Store G3,B3 p128[0] = _mm256_extractf128_ps(p[0], 1); p128[1] = _mm256_extractf128_ps(p[1], 1); @@ -4763,10 +4803,16 @@ inline void rpp_store24_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - _mm_storeu_si128((__m128i*)(dstPtr + 12), px128[0]); - _mm_storeu_si128((__m128i*)(dstPtr + 15), px128[1]); - _mm_storeu_si128((__m128i*)(dstPtr + 18), px128[2]); - _mm_storeu_si128((__m128i*)(dstPtr + 21), px128[3]); + packed01 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed23 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + rpp_storeu_si64(dstPtr + 12, packed01); // Store R4,G4,B4,R5 + rpp_storeu_si32(dstPtr + 16, _mm_srli_si128(packed01, 8)); // Store G5,B5 + rpp_storeu_si64(dstPtr + 18, packed23); // Store R6,G6,B6,R7 + rpp_storeu_si32(dstPtr + 22, _mm_srli_si128(packed23, 8)); // Store G7,B7 } inline void rpp_store8_f32pln1_to_f16pln1_avx(Rpp16f* dstPtr, __m256 p) {