From 2e63d56a33986535e8beb5c8da3a11feb53c2032 Mon Sep 17 00:00:00 2001 From: Manjunath Siddaiah Date: Thu, 27 Aug 2026 15:22:21 -0700 Subject: [PATCH 1/7] Fix for RPP HOST output buffer overrun with NCHW to NHWC conversion --- .../common/cpu/rpp_cpu_simd_load_store.hpp | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp index 6d5de93ed348..6e5c3e0d121a 100644 --- a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp +++ b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp @@ -1884,7 +1884,8 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { _mm_storeu_si128((__m128i*)dstPtr, px128[0]); _mm_storeu_si128((__m128i*)(dstPtr + 3), px128[1]); _mm_storeu_si128((__m128i*)(dstPtr + 6), px128[2]); - _mm_storeu_si128((__m128i*)(dstPtr + 9), px128[3]); + _mm_storel_epi64((__m128i*)(dstPtr + 9), px128[3]); + *((Rpp32u*)(dstPtr + 11)) = _mm_extract_epi32(px128[3], 2); p128[0] = _mm256_extractf128_ps(p[0], 1); p128[1] = _mm256_extractf128_ps(p[2], 1); @@ -1897,7 +1898,8 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { _mm_storeu_si128((__m128i*)(dstPtr + 12), px128[0]); _mm_storeu_si128((__m128i*)(dstPtr + 15), px128[1]); _mm_storeu_si128((__m128i*)(dstPtr + 18), px128[2]); - _mm_storeu_si128((__m128i*)(dstPtr + 21), px128[3]); + _mm_storel_epi64((__m128i*)(dstPtr + 21), px128[3]); + *((Rpp32u*)(dstPtr + 23)) = _mm_extract_epi32(px128[3], 2); p128[0] = _mm256_extractf128_ps(p[1], 0); p128[1] = _mm256_extractf128_ps(p[3], 0); @@ -1910,7 +1912,8 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { _mm_storeu_si128((__m128i*)(dstPtr + 24), px128[0]); _mm_storeu_si128((__m128i*)(dstPtr + 27), px128[1]); _mm_storeu_si128((__m128i*)(dstPtr + 30), px128[2]); - _mm_storeu_si128((__m128i*)(dstPtr + 33), px128[3]); + _mm_storel_epi64((__m128i*)(dstPtr + 33), px128[3]); + *((Rpp32u*)(dstPtr + 35)) = _mm_extract_epi32(px128[3], 2); p128[0] = _mm256_extractf128_ps(p[1], 1); p128[1] = _mm256_extractf128_ps(p[3], 1); @@ -1923,7 +1926,8 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { _mm_storeu_si128((__m128i*)(dstPtr + 36), px128[0]); _mm_storeu_si128((__m128i*)(dstPtr + 39), px128[1]); _mm_storeu_si128((__m128i*)(dstPtr + 42), px128[2]); - _mm_storeu_si128((__m128i*)(dstPtr + 45), px128[3]); + _mm_storel_epi64((__m128i*)(dstPtr + 45), px128[3]); + *((Rpp32u*)(dstPtr + 47)) = _mm_extract_epi32(px128[3], 2); } inline void rpp_load48_u8pln3_to_f32pln3_avx(Rpp8u* srcPtrR, Rpp8u* srcPtrG, Rpp8u* srcPtrB, @@ -2083,9 +2087,9 @@ inline void rpp_store48_f32pln3_to_u8pkd3_avx(Rpp8u* dstPtr, __m256* p) { _mm_storeu_si128( (__m128i*)(dstPtr + 24), px[2]); /* store [R09|G09|B09|R10|G10|B10|R11|G11|B11|R12|G12|B12|00|00|00|00] */ - _mm_storeu_si128( - (__m128i*)(dstPtr + 36), - px[3]); /* store [R13|G13|B13|R14|G14|B14|R15|G15|B15|R16|G16|B16|00|00|00|00] */ + // Fixed: Only store 12 bytes (not 16) to avoid writing beyond 48-byte boundary + _mm_storel_epi64((__m128i*)(dstPtr + 36), px[3]); /* store first 8 bytes [R13|G13|B13|R14|G14|B14|R15|G15] */ + *((Rpp32u*)(dstPtr + 44)) = _mm_extract_epi32(px[3], 2); /* store last 4 bytes [B15|R16|G16|B16] */ } inline void rpp_load24_u8pln3_to_f64pln3_avx(Rpp8u* srcPtrR, Rpp8u* srcPtrG, Rpp8u* srcPtrB, @@ -2596,7 +2600,9 @@ inline void rpp_store24_f32pln3_to_f32pkd3_avx(Rpp32f* dstPtr, __m256* p) { _mm_storeu_ps(dstPtr + 12, p128[0]); _mm_storeu_ps(dstPtr + 15, p128[1]); _mm_storeu_ps(dstPtr + 18, p128[2]); - _mm_storeu_ps(dstPtr + 21, p128[3]); + // Fixed: Only store 3 floats (not 4) to avoid writing beyond 24-float (96-byte) boundary + _mm_storel_pi((__m64*)(dstPtr + 21), p128[3]); /* store first 2 floats [R08|G08] */ + dstPtr[23] = _mm_cvtss_f32(_mm_movehl_ps(p128[3], p128[3])); /* store 3rd float [B08] */ } inline void extract_and_convert_to_f64_pln3_to_pln3(__m256* px, __m256d* p) { @@ -3053,9 +3059,9 @@ inline void rpp_store48_f32pln3_to_i8pkd3_avx(Rpp8s* dstPtr, __m256* p) { _mm_storeu_si128( (__m128i*)(dstPtr + 24), px[2]); /* store [R09|G09|B09|R10|G10|B10|R11|G11|B11|R12|G12|B12|00|00|00|00] */ - _mm_storeu_si128( - (__m128i*)(dstPtr + 36), - px[3]); /* store [R13|G13|B13|R14|G14|B14|R15|G15|B15|R16|G16|B16|00|00|00|00] */ + // Fixed: Only store 12 bytes (not 16) to avoid writing beyond 48-byte boundary + _mm_storel_epi64((__m128i*)(dstPtr + 36), px[3]); /* store first 8 bytes [R13|G13|B13|R14|G14|B14|R15|G15] */ + *((Rpp32u*)(dstPtr + 44)) = _mm_extract_epi32(px[3], 2); /* store last 4 bytes [B15|R16|G16|B16] */ } inline void rpp_load24_i8pln3_to_f64pln3_avx(Rpp8s* srcPtrR, Rpp8s* srcPtrG, Rpp8s* srcPtrB, @@ -4749,10 +4755,14 @@ inline void rpp_store24_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - _mm_storeu_si128((__m128i*)dstPtr, px128[0]); - _mm_storeu_si128((__m128i*)(dstPtr + 3), px128[1]); - _mm_storeu_si128((__m128i*)(dstPtr + 6), px128[2]); - _mm_storeu_si128((__m128i*)(dstPtr + 9), px128[3]); + *((Rpp32u*)(dstPtr + 0)) = _mm_extract_epi32(px128[0], 0); + *((Rpp16u*)(dstPtr + 2)) = (Rpp16u)_mm_extract_epi16(px128[0], 2); + *((Rpp32u*)(dstPtr + 3)) = _mm_extract_epi32(px128[1], 0); + *((Rpp16u*)(dstPtr + 5)) = (Rpp16u)_mm_extract_epi16(px128[1], 2); + *((Rpp32u*)(dstPtr + 6)) = _mm_extract_epi32(px128[2], 0); + *((Rpp16u*)(dstPtr + 8)) = (Rpp16u)_mm_extract_epi16(px128[2], 2); + *((Rpp32u*)(dstPtr + 9)) = _mm_extract_epi32(px128[3], 0); + *((Rpp16u*)(dstPtr + 11)) = (Rpp16u)_mm_extract_epi16(px128[3], 2); p128[0] = _mm256_extractf128_ps(p[0], 1); p128[1] = _mm256_extractf128_ps(p[1], 1); @@ -4763,10 +4773,14 @@ inline void rpp_store24_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - _mm_storeu_si128((__m128i*)(dstPtr + 12), px128[0]); - _mm_storeu_si128((__m128i*)(dstPtr + 15), px128[1]); - _mm_storeu_si128((__m128i*)(dstPtr + 18), px128[2]); - _mm_storeu_si128((__m128i*)(dstPtr + 21), px128[3]); + *((Rpp32u*)(dstPtr + 12)) = _mm_extract_epi32(px128[0], 0); + *((Rpp16u*)(dstPtr + 14)) = (Rpp16u)_mm_extract_epi16(px128[0], 2); + *((Rpp32u*)(dstPtr + 15)) = _mm_extract_epi32(px128[1], 0); + *((Rpp16u*)(dstPtr + 17)) = (Rpp16u)_mm_extract_epi16(px128[1], 2); + *((Rpp32u*)(dstPtr + 18)) = _mm_extract_epi32(px128[2], 0); + *((Rpp16u*)(dstPtr + 20)) = (Rpp16u)_mm_extract_epi16(px128[2], 2); + *((Rpp32u*)(dstPtr + 21)) = _mm_extract_epi32(px128[3], 0); + *((Rpp16u*)(dstPtr + 23)) = (Rpp16u)_mm_extract_epi16(px128[3], 2); } inline void rpp_store8_f32pln1_to_f16pln1_avx(Rpp16f* dstPtr, __m256 p) { From 8a2b0913cb2b0c4eaee9df55263ffaf839eb260a Mon Sep 17 00:00:00 2001 From: Manjunath Siddaiah Date: Thu, 27 Aug 2026 19:57:05 -0700 Subject: [PATCH 2/7] simd data store using utility functions instead of raw pointers dereferencing --- .../common/cpu/rpp_cpu_simd_load_store.hpp | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp index 6e5c3e0d121a..385ce0cf4d29 100644 --- a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp +++ b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp @@ -1885,7 +1885,7 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { _mm_storeu_si128((__m128i*)(dstPtr + 3), px128[1]); _mm_storeu_si128((__m128i*)(dstPtr + 6), px128[2]); _mm_storel_epi64((__m128i*)(dstPtr + 9), px128[3]); - *((Rpp32u*)(dstPtr + 11)) = _mm_extract_epi32(px128[3], 2); + rpp_storeu_si32(dstPtr + 11, _mm_srli_si128(px128[3], 8)); p128[0] = _mm256_extractf128_ps(p[0], 1); p128[1] = _mm256_extractf128_ps(p[2], 1); @@ -1899,7 +1899,7 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { _mm_storeu_si128((__m128i*)(dstPtr + 15), px128[1]); _mm_storeu_si128((__m128i*)(dstPtr + 18), px128[2]); _mm_storel_epi64((__m128i*)(dstPtr + 21), px128[3]); - *((Rpp32u*)(dstPtr + 23)) = _mm_extract_epi32(px128[3], 2); + rpp_storeu_si32(dstPtr + 23, _mm_srli_si128(px128[3], 8)); p128[0] = _mm256_extractf128_ps(p[1], 0); p128[1] = _mm256_extractf128_ps(p[3], 0); @@ -1913,7 +1913,7 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { _mm_storeu_si128((__m128i*)(dstPtr + 27), px128[1]); _mm_storeu_si128((__m128i*)(dstPtr + 30), px128[2]); _mm_storel_epi64((__m128i*)(dstPtr + 33), px128[3]); - *((Rpp32u*)(dstPtr + 35)) = _mm_extract_epi32(px128[3], 2); + rpp_storeu_si32(dstPtr + 35, _mm_srli_si128(px128[3], 8)); p128[0] = _mm256_extractf128_ps(p[1], 1); p128[1] = _mm256_extractf128_ps(p[3], 1); @@ -1927,7 +1927,7 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { _mm_storeu_si128((__m128i*)(dstPtr + 39), px128[1]); _mm_storeu_si128((__m128i*)(dstPtr + 42), px128[2]); _mm_storel_epi64((__m128i*)(dstPtr + 45), px128[3]); - *((Rpp32u*)(dstPtr + 47)) = _mm_extract_epi32(px128[3], 2); + rpp_storeu_si32(dstPtr + 47, _mm_srli_si128(px128[3], 8)); } inline void rpp_load48_u8pln3_to_f32pln3_avx(Rpp8u* srcPtrR, Rpp8u* srcPtrG, Rpp8u* srcPtrB, @@ -2089,7 +2089,7 @@ inline void rpp_store48_f32pln3_to_u8pkd3_avx(Rpp8u* dstPtr, __m256* p) { px[2]); /* store [R09|G09|B09|R10|G10|B10|R11|G11|B11|R12|G12|B12|00|00|00|00] */ // Fixed: Only store 12 bytes (not 16) to avoid writing beyond 48-byte boundary _mm_storel_epi64((__m128i*)(dstPtr + 36), px[3]); /* store first 8 bytes [R13|G13|B13|R14|G14|B14|R15|G15] */ - *((Rpp32u*)(dstPtr + 44)) = _mm_extract_epi32(px[3], 2); /* store last 4 bytes [B15|R16|G16|B16] */ + rpp_storeu_si32(dstPtr + 44, _mm_srli_si128(px[3], 8)); /* store last 4 bytes [B15|R16|G16|B16] */ } inline void rpp_load24_u8pln3_to_f64pln3_avx(Rpp8u* srcPtrR, Rpp8u* srcPtrG, Rpp8u* srcPtrB, @@ -2601,7 +2601,7 @@ inline void rpp_store24_f32pln3_to_f32pkd3_avx(Rpp32f* dstPtr, __m256* p) { _mm_storeu_ps(dstPtr + 15, p128[1]); _mm_storeu_ps(dstPtr + 18, p128[2]); // Fixed: Only store 3 floats (not 4) to avoid writing beyond 24-float (96-byte) boundary - _mm_storel_pi((__m64*)(dstPtr + 21), p128[3]); /* store first 2 floats [R08|G08] */ + rpp_storeu_si64(dstPtr + 21, _mm_castps_si128(p128[3])); /* store first 2 floats [R08|G08] */ dstPtr[23] = _mm_cvtss_f32(_mm_movehl_ps(p128[3], p128[3])); /* store 3rd float [B08] */ } @@ -3061,7 +3061,7 @@ inline void rpp_store48_f32pln3_to_i8pkd3_avx(Rpp8s* dstPtr, __m256* p) { px[2]); /* store [R09|G09|B09|R10|G10|B10|R11|G11|B11|R12|G12|B12|00|00|00|00] */ // Fixed: Only store 12 bytes (not 16) to avoid writing beyond 48-byte boundary _mm_storel_epi64((__m128i*)(dstPtr + 36), px[3]); /* store first 8 bytes [R13|G13|B13|R14|G14|B14|R15|G15] */ - *((Rpp32u*)(dstPtr + 44)) = _mm_extract_epi32(px[3], 2); /* store last 4 bytes [B15|R16|G16|B16] */ + rpp_storeu_si32(dstPtr + 44, _mm_srli_si128(px[3], 8)); /* store last 4 bytes [B15|R16|G16|B16] */ } inline void rpp_load24_i8pln3_to_f64pln3_avx(Rpp8s* srcPtrR, Rpp8s* srcPtrG, Rpp8s* srcPtrB, @@ -4755,14 +4755,15 @@ inline void rpp_store24_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - *((Rpp32u*)(dstPtr + 0)) = _mm_extract_epi32(px128[0], 0); - *((Rpp16u*)(dstPtr + 2)) = (Rpp16u)_mm_extract_epi16(px128[0], 2); - *((Rpp32u*)(dstPtr + 3)) = _mm_extract_epi32(px128[1], 0); - *((Rpp16u*)(dstPtr + 5)) = (Rpp16u)_mm_extract_epi16(px128[1], 2); - *((Rpp32u*)(dstPtr + 6)) = _mm_extract_epi32(px128[2], 0); - *((Rpp16u*)(dstPtr + 8)) = (Rpp16u)_mm_extract_epi16(px128[2], 2); - *((Rpp32u*)(dstPtr + 9)) = _mm_extract_epi32(px128[3], 0); - *((Rpp16u*)(dstPtr + 11)) = (Rpp16u)_mm_extract_epi16(px128[3], 2); + // Each px128[i] contains [Ri, Gi, Bi, unused] - use overlapping 32-bit stores to write [R,G] then [G,B] + rpp_storeu_si32(dstPtr + 0, px128[0]); // Store R0, G0 + rpp_storeu_si32(dstPtr + 1, _mm_srli_si128(px128[0], 2)); // Store G0, B0 (G0 overlaps) + rpp_storeu_si32(dstPtr + 3, px128[1]); // Store R1, G1 + rpp_storeu_si32(dstPtr + 4, _mm_srli_si128(px128[1], 2)); // Store G1, B1 (G1 overlaps) + rpp_storeu_si32(dstPtr + 6, px128[2]); // Store R2, G2 + rpp_storeu_si32(dstPtr + 7, _mm_srli_si128(px128[2], 2)); // Store G2, B2 (G2 overlaps) + rpp_storeu_si32(dstPtr + 9, px128[3]); // Store R3, G3 + rpp_storeu_si32(dstPtr + 10, _mm_srli_si128(px128[3], 2)); // Store G3, B3 (G3 overlaps) p128[0] = _mm256_extractf128_ps(p[0], 1); p128[1] = _mm256_extractf128_ps(p[1], 1); @@ -4773,14 +4774,15 @@ inline void rpp_store24_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - *((Rpp32u*)(dstPtr + 12)) = _mm_extract_epi32(px128[0], 0); - *((Rpp16u*)(dstPtr + 14)) = (Rpp16u)_mm_extract_epi16(px128[0], 2); - *((Rpp32u*)(dstPtr + 15)) = _mm_extract_epi32(px128[1], 0); - *((Rpp16u*)(dstPtr + 17)) = (Rpp16u)_mm_extract_epi16(px128[1], 2); - *((Rpp32u*)(dstPtr + 18)) = _mm_extract_epi32(px128[2], 0); - *((Rpp16u*)(dstPtr + 20)) = (Rpp16u)_mm_extract_epi16(px128[2], 2); - *((Rpp32u*)(dstPtr + 21)) = _mm_extract_epi32(px128[3], 0); - *((Rpp16u*)(dstPtr + 23)) = (Rpp16u)_mm_extract_epi16(px128[3], 2); + // Each px128[i] contains [Ri, Gi, Bi, unused] - use overlapping 32-bit stores to write [R,G] then [G,B] + rpp_storeu_si32(dstPtr + 12, px128[0]); // Store R4, G4 + rpp_storeu_si32(dstPtr + 13, _mm_srli_si128(px128[0], 2)); // Store G4, B4 (G4 overlaps) + rpp_storeu_si32(dstPtr + 15, px128[1]); // Store R5, G5 + rpp_storeu_si32(dstPtr + 16, _mm_srli_si128(px128[1], 2)); // Store G5, B5 (G5 overlaps) + rpp_storeu_si32(dstPtr + 18, px128[2]); // Store R6, G6 + rpp_storeu_si32(dstPtr + 19, _mm_srli_si128(px128[2], 2)); // Store G6, B6 (G6 overlaps) + rpp_storeu_si32(dstPtr + 21, px128[3]); // Store R7, G7 + rpp_storeu_si32(dstPtr + 22, _mm_srli_si128(px128[3], 2)); // Store G7, B7 (G7 overlaps) } inline void rpp_store8_f32pln1_to_f16pln1_avx(Rpp16f* dstPtr, __m256 p) { From 1aabf39effb86c7ad676365504c52329c5088f17 Mon Sep 17 00:00:00 2001 From: Manjunath Siddaiah Date: Fri, 28 Aug 2026 09:06:34 -0700 Subject: [PATCH 3/7] Fix for pre-commit clang-format --- .../common/cpu/rpp_cpu_simd_load_store.hpp | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp index 385ce0cf4d29..66846026ca4f 100644 --- a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp +++ b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp @@ -2088,8 +2088,10 @@ inline void rpp_store48_f32pln3_to_u8pkd3_avx(Rpp8u* dstPtr, __m256* p) { (__m128i*)(dstPtr + 24), px[2]); /* store [R09|G09|B09|R10|G10|B10|R11|G11|B11|R12|G12|B12|00|00|00|00] */ // Fixed: Only store 12 bytes (not 16) to avoid writing beyond 48-byte boundary - _mm_storel_epi64((__m128i*)(dstPtr + 36), px[3]); /* store first 8 bytes [R13|G13|B13|R14|G14|B14|R15|G15] */ - rpp_storeu_si32(dstPtr + 44, _mm_srli_si128(px[3], 8)); /* store last 4 bytes [B15|R16|G16|B16] */ + _mm_storel_epi64((__m128i*)(dstPtr + 36), + px[3]); /* store first 8 bytes [R13|G13|B13|R14|G14|B14|R15|G15] */ + rpp_storeu_si32(dstPtr + 44, + _mm_srli_si128(px[3], 8)); /* store last 4 bytes [B15|R16|G16|B16] */ } inline void rpp_load24_u8pln3_to_f64pln3_avx(Rpp8u* srcPtrR, Rpp8u* srcPtrG, Rpp8u* srcPtrB, @@ -3060,8 +3062,10 @@ inline void rpp_store48_f32pln3_to_i8pkd3_avx(Rpp8s* dstPtr, __m256* p) { (__m128i*)(dstPtr + 24), px[2]); /* store [R09|G09|B09|R10|G10|B10|R11|G11|B11|R12|G12|B12|00|00|00|00] */ // Fixed: Only store 12 bytes (not 16) to avoid writing beyond 48-byte boundary - _mm_storel_epi64((__m128i*)(dstPtr + 36), px[3]); /* store first 8 bytes [R13|G13|B13|R14|G14|B14|R15|G15] */ - rpp_storeu_si32(dstPtr + 44, _mm_srli_si128(px[3], 8)); /* store last 4 bytes [B15|R16|G16|B16] */ + _mm_storel_epi64((__m128i*)(dstPtr + 36), + px[3]); /* store first 8 bytes [R13|G13|B13|R14|G14|B14|R15|G15] */ + rpp_storeu_si32(dstPtr + 44, + _mm_srli_si128(px[3], 8)); /* store last 4 bytes [B15|R16|G16|B16] */ } inline void rpp_load24_i8pln3_to_f64pln3_avx(Rpp8s* srcPtrR, Rpp8s* srcPtrG, Rpp8s* srcPtrB, @@ -4755,15 +4759,16 @@ inline void rpp_store24_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - // Each px128[i] contains [Ri, Gi, Bi, unused] - use overlapping 32-bit stores to write [R,G] then [G,B] - rpp_storeu_si32(dstPtr + 0, px128[0]); // Store R0, G0 - rpp_storeu_si32(dstPtr + 1, _mm_srli_si128(px128[0], 2)); // Store G0, B0 (G0 overlaps) - rpp_storeu_si32(dstPtr + 3, px128[1]); // Store R1, G1 - rpp_storeu_si32(dstPtr + 4, _mm_srli_si128(px128[1], 2)); // Store G1, B1 (G1 overlaps) - rpp_storeu_si32(dstPtr + 6, px128[2]); // Store R2, G2 - rpp_storeu_si32(dstPtr + 7, _mm_srli_si128(px128[2], 2)); // Store G2, B2 (G2 overlaps) - rpp_storeu_si32(dstPtr + 9, px128[3]); // Store R3, G3 - rpp_storeu_si32(dstPtr + 10, _mm_srli_si128(px128[3], 2)); // Store G3, B3 (G3 overlaps) + // Each px128[i] contains [Ri, Gi, Bi, unused] - use overlapping 32-bit stores to write [R,G] + // then [G,B] + rpp_storeu_si32(dstPtr + 0, px128[0]); // Store R0, G0 + rpp_storeu_si32(dstPtr + 1, _mm_srli_si128(px128[0], 2)); // Store G0, B0 (G0 overlaps) + rpp_storeu_si32(dstPtr + 3, px128[1]); // Store R1, G1 + rpp_storeu_si32(dstPtr + 4, _mm_srli_si128(px128[1], 2)); // Store G1, B1 (G1 overlaps) + rpp_storeu_si32(dstPtr + 6, px128[2]); // Store R2, G2 + rpp_storeu_si32(dstPtr + 7, _mm_srli_si128(px128[2], 2)); // Store G2, B2 (G2 overlaps) + rpp_storeu_si32(dstPtr + 9, px128[3]); // Store R3, G3 + rpp_storeu_si32(dstPtr + 10, _mm_srli_si128(px128[3], 2)); // Store G3, B3 (G3 overlaps) p128[0] = _mm256_extractf128_ps(p[0], 1); p128[1] = _mm256_extractf128_ps(p[1], 1); @@ -4774,15 +4779,16 @@ inline void rpp_store24_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - // Each px128[i] contains [Ri, Gi, Bi, unused] - use overlapping 32-bit stores to write [R,G] then [G,B] - rpp_storeu_si32(dstPtr + 12, px128[0]); // Store R4, G4 - rpp_storeu_si32(dstPtr + 13, _mm_srli_si128(px128[0], 2)); // Store G4, B4 (G4 overlaps) - rpp_storeu_si32(dstPtr + 15, px128[1]); // Store R5, G5 - rpp_storeu_si32(dstPtr + 16, _mm_srli_si128(px128[1], 2)); // Store G5, B5 (G5 overlaps) - rpp_storeu_si32(dstPtr + 18, px128[2]); // Store R6, G6 - rpp_storeu_si32(dstPtr + 19, _mm_srli_si128(px128[2], 2)); // Store G6, B6 (G6 overlaps) - rpp_storeu_si32(dstPtr + 21, px128[3]); // Store R7, G7 - rpp_storeu_si32(dstPtr + 22, _mm_srli_si128(px128[3], 2)); // Store G7, B7 (G7 overlaps) + // Each px128[i] contains [Ri, Gi, Bi, unused] - use overlapping 32-bit stores to write [R,G] + // then [G,B] + rpp_storeu_si32(dstPtr + 12, px128[0]); // Store R4, G4 + rpp_storeu_si32(dstPtr + 13, _mm_srli_si128(px128[0], 2)); // Store G4, B4 (G4 overlaps) + rpp_storeu_si32(dstPtr + 15, px128[1]); // Store R5, G5 + rpp_storeu_si32(dstPtr + 16, _mm_srli_si128(px128[1], 2)); // Store G5, B5 (G5 overlaps) + rpp_storeu_si32(dstPtr + 18, px128[2]); // Store R6, G6 + rpp_storeu_si32(dstPtr + 19, _mm_srli_si128(px128[2], 2)); // Store G6, B6 (G6 overlaps) + rpp_storeu_si32(dstPtr + 21, px128[3]); // Store R7, G7 + rpp_storeu_si32(dstPtr + 22, _mm_srli_si128(px128[3], 2)); // Store G7, B7 (G7 overlaps) } inline void rpp_store8_f32pln1_to_f16pln1_avx(Rpp16f* dstPtr, __m256 p) { From 138dfc1aa2e936947fbd539d7790c51a6a39eaa0 Mon Sep 17 00:00:00 2001 From: Manjunath Siddaiah Date: Wed, 2 Sep 2026 09:15:29 -0700 Subject: [PATCH 4/7] Optimized store24 and store48 for F16 data type with PLN3 to PKD3 conversion --- .../common/cpu/rpp_cpu_simd_load_store.hpp | 102 +++++++++++------- 1 file changed, 62 insertions(+), 40 deletions(-) diff --git a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp index 66846026ca4f..adfcc0a10bda 100644 --- a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp +++ b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp @@ -1881,11 +1881,16 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - _mm_storeu_si128((__m128i*)dstPtr, px128[0]); - _mm_storeu_si128((__m128i*)(dstPtr + 3), px128[1]); - _mm_storeu_si128((__m128i*)(dstPtr + 6), px128[2]); - _mm_storel_epi64((__m128i*)(dstPtr + 9), px128[3]); - rpp_storeu_si32(dstPtr + 11, _mm_srli_si128(px128[3], 8)); + __m128i packed01 = _mm_shuffle_epi8( + _mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + __m128i packed23 = _mm_shuffle_epi8( + _mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + rpp_storeu_si64(dstPtr + 0, packed01); + rpp_storeu_si32(dstPtr + 4, _mm_srli_si128(packed01, 8)); + rpp_storeu_si64(dstPtr + 6, packed23); + rpp_storeu_si32(dstPtr + 10, _mm_srli_si128(packed23, 8)); p128[0] = _mm256_extractf128_ps(p[0], 1); p128[1] = _mm256_extractf128_ps(p[2], 1); @@ -1895,11 +1900,16 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - _mm_storeu_si128((__m128i*)(dstPtr + 12), px128[0]); - _mm_storeu_si128((__m128i*)(dstPtr + 15), px128[1]); - _mm_storeu_si128((__m128i*)(dstPtr + 18), px128[2]); - _mm_storel_epi64((__m128i*)(dstPtr + 21), px128[3]); - rpp_storeu_si32(dstPtr + 23, _mm_srli_si128(px128[3], 8)); + packed01 = _mm_shuffle_epi8( + _mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed23 = _mm_shuffle_epi8( + _mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + rpp_storeu_si64(dstPtr + 12, packed01); + rpp_storeu_si32(dstPtr + 16, _mm_srli_si128(packed01, 8)); + rpp_storeu_si64(dstPtr + 18, packed23); + rpp_storeu_si32(dstPtr + 22, _mm_srli_si128(packed23, 8)); p128[0] = _mm256_extractf128_ps(p[1], 0); p128[1] = _mm256_extractf128_ps(p[3], 0); @@ -1909,11 +1919,16 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - _mm_storeu_si128((__m128i*)(dstPtr + 24), px128[0]); - _mm_storeu_si128((__m128i*)(dstPtr + 27), px128[1]); - _mm_storeu_si128((__m128i*)(dstPtr + 30), px128[2]); - _mm_storel_epi64((__m128i*)(dstPtr + 33), px128[3]); - rpp_storeu_si32(dstPtr + 35, _mm_srli_si128(px128[3], 8)); + packed01 = _mm_shuffle_epi8( + _mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed23 = _mm_shuffle_epi8( + _mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + rpp_storeu_si64(dstPtr + 24, packed01); + rpp_storeu_si32(dstPtr + 28, _mm_srli_si128(packed01, 8)); + rpp_storeu_si64(dstPtr + 30, packed23); + rpp_storeu_si32(dstPtr + 34, _mm_srli_si128(packed23, 8)); p128[0] = _mm256_extractf128_ps(p[1], 1); p128[1] = _mm256_extractf128_ps(p[3], 1); @@ -1923,11 +1938,16 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - _mm_storeu_si128((__m128i*)(dstPtr + 36), px128[0]); - _mm_storeu_si128((__m128i*)(dstPtr + 39), px128[1]); - _mm_storeu_si128((__m128i*)(dstPtr + 42), px128[2]); - _mm_storel_epi64((__m128i*)(dstPtr + 45), px128[3]); - rpp_storeu_si32(dstPtr + 47, _mm_srli_si128(px128[3], 8)); + packed01 = _mm_shuffle_epi8( + _mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed23 = _mm_shuffle_epi8( + _mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + rpp_storeu_si64(dstPtr + 36, packed01); + rpp_storeu_si32(dstPtr + 40, _mm_srli_si128(packed01, 8)); + rpp_storeu_si64(dstPtr + 42, packed23); + rpp_storeu_si32(dstPtr + 46, _mm_srli_si128(packed23, 8)); } inline void rpp_load48_u8pln3_to_f32pln3_avx(Rpp8u* srcPtrR, Rpp8u* srcPtrG, Rpp8u* srcPtrB, @@ -4759,16 +4779,18 @@ inline void rpp_store24_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - // Each px128[i] contains [Ri, Gi, Bi, unused] - use overlapping 32-bit stores to write [R,G] - // then [G,B] - rpp_storeu_si32(dstPtr + 0, px128[0]); // Store R0, G0 - rpp_storeu_si32(dstPtr + 1, _mm_srli_si128(px128[0], 2)); // Store G0, B0 (G0 overlaps) - rpp_storeu_si32(dstPtr + 3, px128[1]); // Store R1, G1 - rpp_storeu_si32(dstPtr + 4, _mm_srli_si128(px128[1], 2)); // Store G1, B1 (G1 overlaps) - rpp_storeu_si32(dstPtr + 6, px128[2]); // Store R2, G2 - rpp_storeu_si32(dstPtr + 7, _mm_srli_si128(px128[2], 2)); // Store G2, B2 (G2 overlaps) - rpp_storeu_si32(dstPtr + 9, px128[3]); // Store R3, G3 - rpp_storeu_si32(dstPtr + 10, _mm_srli_si128(px128[3], 2)); // Store G3, B3 (G3 overlaps) + // Each px128[i] contains [Ri, Gi, Bi, unused] - pack pairs without unused elements + // Shuffle to pack: px128[0]+px128[1] -> [R0,G0,B0,R1,G1,B1,X,X] + __m128i packed01 = _mm_shuffle_epi8( + _mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + __m128i packed23 = _mm_shuffle_epi8( + _mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + rpp_storeu_si64(dstPtr + 0, packed01); // Store R0,G0,B0,R1 + rpp_storeu_si32(dstPtr + 4, _mm_srli_si128(packed01, 8)); // Store G1,B1 + rpp_storeu_si64(dstPtr + 6, packed23); // Store R2,G2,B2,R3 + rpp_storeu_si32(dstPtr + 10, _mm_srli_si128(packed23, 8)); // Store G3,B3 p128[0] = _mm256_extractf128_ps(p[0], 1); p128[1] = _mm256_extractf128_ps(p[1], 1); @@ -4779,16 +4801,16 @@ inline void rpp_store24_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - // Each px128[i] contains [Ri, Gi, Bi, unused] - use overlapping 32-bit stores to write [R,G] - // then [G,B] - rpp_storeu_si32(dstPtr + 12, px128[0]); // Store R4, G4 - rpp_storeu_si32(dstPtr + 13, _mm_srli_si128(px128[0], 2)); // Store G4, B4 (G4 overlaps) - rpp_storeu_si32(dstPtr + 15, px128[1]); // Store R5, G5 - rpp_storeu_si32(dstPtr + 16, _mm_srli_si128(px128[1], 2)); // Store G5, B5 (G5 overlaps) - rpp_storeu_si32(dstPtr + 18, px128[2]); // Store R6, G6 - rpp_storeu_si32(dstPtr + 19, _mm_srli_si128(px128[2], 2)); // Store G6, B6 (G6 overlaps) - rpp_storeu_si32(dstPtr + 21, px128[3]); // Store R7, G7 - rpp_storeu_si32(dstPtr + 22, _mm_srli_si128(px128[3], 2)); // Store G7, B7 (G7 overlaps) + packed01 = _mm_shuffle_epi8( + _mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed23 = _mm_shuffle_epi8( + _mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + rpp_storeu_si64(dstPtr + 12, packed01); // Store R4,G4,B4,R5 + rpp_storeu_si32(dstPtr + 16, _mm_srli_si128(packed01, 8)); // Store G5,B5 + rpp_storeu_si64(dstPtr + 18, packed23); // Store R6,G6,B6,R7 + rpp_storeu_si32(dstPtr + 22, _mm_srli_si128(packed23, 8)); // Store G7,B7 } inline void rpp_store8_f32pln1_to_f16pln1_avx(Rpp16f* dstPtr, __m256 p) { From b730567f275a9e994f6004d211f14392fa14e130 Mon Sep 17 00:00:00 2001 From: Manjunath Siddaiah Date: Wed, 2 Sep 2026 09:23:24 -0700 Subject: [PATCH 5/7] Fix for pre-commit --- .../common/cpu/rpp_cpu_simd_load_store.hpp | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp index adfcc0a10bda..e8ef3f444536 100644 --- a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp +++ b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp @@ -1881,12 +1881,12 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - __m128i packed01 = _mm_shuffle_epi8( - _mm_unpacklo_epi64(px128[0], px128[1]), - _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); - __m128i packed23 = _mm_shuffle_epi8( - _mm_unpacklo_epi64(px128[2], px128[3]), - _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + __m128i packed01 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + __m128i packed23 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); rpp_storeu_si64(dstPtr + 0, packed01); rpp_storeu_si32(dstPtr + 4, _mm_srli_si128(packed01, 8)); rpp_storeu_si64(dstPtr + 6, packed23); @@ -1900,12 +1900,12 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - packed01 = _mm_shuffle_epi8( - _mm_unpacklo_epi64(px128[0], px128[1]), - _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); - packed23 = _mm_shuffle_epi8( - _mm_unpacklo_epi64(px128[2], px128[3]), - _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed01 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed23 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); rpp_storeu_si64(dstPtr + 12, packed01); rpp_storeu_si32(dstPtr + 16, _mm_srli_si128(packed01, 8)); rpp_storeu_si64(dstPtr + 18, packed23); @@ -1919,12 +1919,12 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - packed01 = _mm_shuffle_epi8( - _mm_unpacklo_epi64(px128[0], px128[1]), - _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); - packed23 = _mm_shuffle_epi8( - _mm_unpacklo_epi64(px128[2], px128[3]), - _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed01 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed23 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); rpp_storeu_si64(dstPtr + 24, packed01); rpp_storeu_si32(dstPtr + 28, _mm_srli_si128(packed01, 8)); rpp_storeu_si64(dstPtr + 30, packed23); @@ -1938,12 +1938,12 @@ inline void rpp_store48_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - packed01 = _mm_shuffle_epi8( - _mm_unpacklo_epi64(px128[0], px128[1]), - _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); - packed23 = _mm_shuffle_epi8( - _mm_unpacklo_epi64(px128[2], px128[3]), - _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed01 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed23 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); rpp_storeu_si64(dstPtr + 36, packed01); rpp_storeu_si32(dstPtr + 40, _mm_srli_si128(packed01, 8)); rpp_storeu_si64(dstPtr + 42, packed23); @@ -4781,12 +4781,12 @@ inline void rpp_store24_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); // Each px128[i] contains [Ri, Gi, Bi, unused] - pack pairs without unused elements // Shuffle to pack: px128[0]+px128[1] -> [R0,G0,B0,R1,G1,B1,X,X] - __m128i packed01 = _mm_shuffle_epi8( - _mm_unpacklo_epi64(px128[0], px128[1]), - _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); - __m128i packed23 = _mm_shuffle_epi8( - _mm_unpacklo_epi64(px128[2], px128[3]), - _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + __m128i packed01 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + __m128i packed23 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); rpp_storeu_si64(dstPtr + 0, packed01); // Store R0,G0,B0,R1 rpp_storeu_si32(dstPtr + 4, _mm_srli_si128(packed01, 8)); // Store G1,B1 rpp_storeu_si64(dstPtr + 6, packed23); // Store R2,G2,B2,R3 @@ -4801,12 +4801,12 @@ inline void rpp_store24_f32pln3_to_f16pkd3_avx(Rpp16f* dstPtr, __m256* p) { px128[1] = _mm_cvtps_ph(p128[1], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[2] = _mm_cvtps_ph(p128[2], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); px128[3] = _mm_cvtps_ph(p128[3], _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC); - packed01 = _mm_shuffle_epi8( - _mm_unpacklo_epi64(px128[0], px128[1]), - _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); - packed23 = _mm_shuffle_epi8( - _mm_unpacklo_epi64(px128[2], px128[3]), - _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed01 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[0], px128[1]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); + packed23 = + _mm_shuffle_epi8(_mm_unpacklo_epi64(px128[2], px128[3]), + _mm_setr_epi8(0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1)); rpp_storeu_si64(dstPtr + 12, packed01); // Store R4,G4,B4,R5 rpp_storeu_si32(dstPtr + 16, _mm_srli_si128(packed01, 8)); // Store G5,B5 rpp_storeu_si64(dstPtr + 18, packed23); // Store R6,G6,B6,R7 From b1c31f8a9cc2cea9c854789cf5356376295b4f1f Mon Sep 17 00:00:00 2001 From: Manjunath Siddaiah Date: Wed, 2 Sep 2026 16:34:49 -0700 Subject: [PATCH 6/7] Optimized store for f32 pln3 to pkd3 conversion --- projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp index e8ef3f444536..98097b428948 100644 --- a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp +++ b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp @@ -2624,7 +2624,7 @@ inline void rpp_store24_f32pln3_to_f32pkd3_avx(Rpp32f* dstPtr, __m256* p) { _mm_storeu_ps(dstPtr + 18, p128[2]); // Fixed: Only store 3 floats (not 4) to avoid writing beyond 24-float (96-byte) boundary rpp_storeu_si64(dstPtr + 21, _mm_castps_si128(p128[3])); /* store first 2 floats [R08|G08] */ - dstPtr[23] = _mm_cvtss_f32(_mm_movehl_ps(p128[3], p128[3])); /* store 3rd float [B08] */ + _mm_store_ss(dstPtr + 23, _mm_castsi128_ps(_mm_srli_si128(_mm_castps_si128(p128[3]), 8))); /* store 3rd float [B08] using shift */ } inline void extract_and_convert_to_f64_pln3_to_pln3(__m256* px, __m256d* p) { From d879105b39566ca2c5680f2a6e2df83ba7165471 Mon Sep 17 00:00:00 2001 From: Manjunath Siddaiah Date: Wed, 2 Sep 2026 16:44:39 -0700 Subject: [PATCH 7/7] Fix clang-format for F32 PKD3 store optimization --- .../rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp index 98097b428948..a3db12bb4975 100644 --- a/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp +++ b/projects/rpp/src/include/common/cpu/rpp_cpu_simd_load_store.hpp @@ -2624,7 +2624,9 @@ inline void rpp_store24_f32pln3_to_f32pkd3_avx(Rpp32f* dstPtr, __m256* p) { _mm_storeu_ps(dstPtr + 18, p128[2]); // Fixed: Only store 3 floats (not 4) to avoid writing beyond 24-float (96-byte) boundary rpp_storeu_si64(dstPtr + 21, _mm_castps_si128(p128[3])); /* store first 2 floats [R08|G08] */ - _mm_store_ss(dstPtr + 23, _mm_castsi128_ps(_mm_srli_si128(_mm_castps_si128(p128[3]), 8))); /* store 3rd float [B08] using shift */ + _mm_store_ss(dstPtr + 23, + _mm_castsi128_ps(_mm_srli_si128(_mm_castps_si128(p128[3]), + 8))); /* store 3rd float [B08] using shift */ } inline void extract_and_convert_to_f64_pln3_to_pln3(__m256* px, __m256d* p) {