diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index 29ff1780f12e2..f1b3b091bbc6e 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -590,6 +590,7 @@ else() COMPILES_P10 ) if(COMPILES_P10) + enable_language(ASM) check_cxx_source_compiles(" #ifdef _AIX #define POWER_10 0x40000 @@ -619,6 +620,11 @@ else() ${MLAS_SRC_DIR}/power/DgemmKernelPOWER10.cpp ${MLAS_SRC_DIR}/power/qgemm_kernel_power10.cpp ) + # Only compile assembly on non-AIX systems + if (NOT AIX) + list(APPEND mlas_platform_srcs_power10 ${MLAS_SRC_DIR}/power/SgemmKernelPackA.S) + set_source_files_properties(${MLAS_SRC_DIR}/power/SgemmKernelPackA.S PROPERTIES COMPILE_FLAGS "-O2 -mcpu=power10") + endif() set_source_files_properties(${MLAS_SRC_DIR}/power/SgemmKernelPOWER10.cpp PROPERTIES COMPILE_FLAGS "-O2 -mcpu=power10 -DSINGLE") set_source_files_properties(${MLAS_SRC_DIR}/power/DgemmKernelPOWER10.cpp PROPERTIES COMPILE_FLAGS "-O2 -mcpu=power10") set_source_files_properties(${MLAS_SRC_DIR}/power/qgemm_kernel_power10.cpp PROPERTIES COMPILE_FLAGS "-O3 -mcpu=power10") diff --git a/onnxruntime/core/mlas/lib/power/SgemmKernelPOWER10.cpp b/onnxruntime/core/mlas/lib/power/SgemmKernelPOWER10.cpp index 3dfe061c72524..9ecfa3d984c64 100644 --- a/onnxruntime/core/mlas/lib/power/SgemmKernelPOWER10.cpp +++ b/onnxruntime/core/mlas/lib/power/SgemmKernelPOWER10.cpp @@ -15,7 +15,12 @@ Module Name: --*/ +#define PREFETCH_ADDR(addr) \ + asm volatile("dcbt 0, %0" ::"r"(addr) : "memory"); + #include "SgemmKernelpower.h" +extern "C" void +PackAKernelPOWER10(__vector float* D, const float* A, size_t lda, size_t k, size_t RowCount); struct MlasSgemmBroadcastAElementsMMA { template @@ -28,7 +33,7 @@ struct MlasSgemmBroadcastAElementsMMA size_t lda ) { - ABroadcast[0][Row] = A [Row * lda]; + ABroadcast[0] = vec_insert(A[Row * lda], ABroadcast[0], Row); } }; @@ -143,32 +148,28 @@ struct MlasSgemmStoreScalarMMA } }; -template +template MLAS_FORCEINLINE -size_t -MlasSgemmMMAProcessCount( - const float* A, - const float* B, - float* C, - size_t CountM, - size_t CountK, - size_t CountN, - size_t lda, - size_t ldc, - MLAS_FLOAT32X4 AlphaBroadcast, - bool ZeroMode + size_t + MlasSgemmMMAProcessCount( + __vector float* Pa, + const float* B, + float* C, + size_t CountM, + size_t CountK, + size_t CountN, + size_t ldc, + MLAS_FLOAT32X4 AlphaBroadcast, + bool ZeroMode ) { do { - - const float* a = A; + __vector float* pa1 = Pa; size_t k = CountK; - MLAS_FLOAT32X4 Accumulators[2][RowCount] = {{ 0 }}; + MLAS_FLOAT32X4 Accumulators[2][RowCount] = {{0}}; MLAS_FLOAT32X4 Result[RowCount]; - MLAS_FLOAT32X4 AElements[RowCount]; - MLAS_FLOAT32X4 ABroadcast[RowCount] = { 0 }; - MLAS_FLOAT32X4 A2Broadcast[RowCount] = { 0 }; + MLAS_FLOAT32X4 ABroadcast[RowCount] = {0}; __vector_quad acc[8]; // @@ -186,30 +187,61 @@ MlasSgemmMMAProcessCount( // // Compute the output block. // - while (k >= 4) { - - MlasLoopUnroll()(AElements, a, lda); - MlasSgemmComputeAElements(AElements, ABroadcast); + while (k >= 8) { if (CountM == 8) { - MlasLoopUnroll()(AElements, a + ( lda * 4), lda); - MlasSgemmComputeAElements(AElements, A2Broadcast); + MlasSgemmComputeBlockMMA(&acc[0], pa1[0], pa1[4], B, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[1], pa1[5], B + 16, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[2], pa1[6], B + 32, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[3], pa1[7], B + 48, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[8], pa1[12], B + 64, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[9], pa1[13], B + 80, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[10], pa1[14], B + 96, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[11], pa1[15], B + 112, CountM); + B += 128; + pa1 += 16; + k -= 8; + } else { + MlasSgemmComputeBlockMMA(&acc[0], pa1[0], ABroadcast[0], B, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[1], ABroadcast[1], B + 16, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[2], ABroadcast[2], B + 32, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[3], ABroadcast[3], B + 48, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[4], ABroadcast[0], B + 64, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[5], ABroadcast[1], B + 80, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[6], ABroadcast[2], B + 96, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[7], ABroadcast[3], B + 112, CountM); + B += 128; + pa1 += 8; + k -= 8; } - MlasSgemmComputeBlockMMA(&acc[0], ABroadcast[0], A2Broadcast[0], B, CountM); - MlasSgemmComputeBlockMMA(&acc[0], ABroadcast[1], A2Broadcast[1], B+16, CountM); - MlasSgemmComputeBlockMMA(&acc[0], ABroadcast[2], A2Broadcast[2], B+32, CountM); - MlasSgemmComputeBlockMMA(&acc[0], ABroadcast[3], A2Broadcast[3], B+48, CountM); - B += 16 * 4; - a += 4; - k -= 4; } + while (k >= 4) { + if (CountM == 8) { + MlasSgemmComputeBlockMMA(&acc[0], pa1[0], pa1[4], B, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[1], pa1[5], B + 16, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[2], pa1[6], B + 32, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[3], pa1[7], B + 48, CountM); + B += 16 * 4; + pa1 += 8; + k -= 4; + } else { + MlasSgemmComputeBlockMMA(&acc[0], pa1[0], ABroadcast[0], B, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[1], ABroadcast[1], B + 16, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[2], ABroadcast[2], B + 32, CountM); + MlasSgemmComputeBlockMMA(&acc[0], pa1[3], ABroadcast[3], B + 48, CountM); + B += 16 * 4; + pa1 += 4; + k -= 4; + } + } while (k > 0) { - MlasLoopUnroll()(ABroadcast, a, lda); - if (CountM == 8) { - MlasLoopUnroll()(A2Broadcast, a + (lda * 4), lda); + if (CountM == 8) { + MlasSgemmComputeBlockMMA(&acc[0], pa1[0], pa1[1], B, CountM); + pa1 += 2; + } else { + MlasSgemmComputeBlockMMA(&acc[0], pa1[0], ABroadcast[0], B, CountM); + pa1 += 1; } - MlasSgemmComputeBlockMMA(&acc[0], ABroadcast[0], A2Broadcast[0], B, CountM); - a += 1; B += 16; k -= 1; } @@ -340,6 +372,236 @@ MlasSgemmMMAProcessCount( return CountM; } +template +MLAS_FORCEINLINE void +MlasSgemmPackA( + __vector float* D, + const float* A, + size_t lda, + size_t k +) +{ + __vector float a1, a2; + const float* a = A; + MLAS_FLOAT32X4 AElements[RowCount] = {}; + MLAS_FLOAT32X4 A2Elements[RowCount] = {}; + while (k >= 16) + + { + PREFETCH_ADDR(a); + PREFETCH_ADDR(a + lda); + PREFETCH_ADDR(a + 2 * lda); + PREFETCH_ADDR(a + 3 * lda); + PREFETCH_ADDR(a + 4 * lda); + PREFETCH_ADDR(a + 5 * lda); + PREFETCH_ADDR(a + 6 * lda); + PREFETCH_ADDR(a + 7 * lda); + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(AElements, a, lda); + a1 = vec_mergee(AElements[0], AElements[1]); + a2 = vec_mergee(AElements[2], AElements[3]); + D[0] = vec_xxpermdi(a1, a2, 0); + D[2] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(AElements[0], AElements[1]); + a2 = vec_mergeo(AElements[2], AElements[3]); + D[1] = vec_xxpermdi(a1, a2, 0); + D[3] = vec_xxpermdi(a1, a2, 3); + if (RowCount == 8) { + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(AElements, a + 4, lda); + a1 = vec_mergee(AElements[0], AElements[1]); + a2 = vec_mergee(AElements[2], AElements[3]); + D[8] = vec_xxpermdi(a1, a2, 0); + D[10] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(AElements[0], AElements[1]); + a2 = vec_mergeo(AElements[2], AElements[3]); + D[9] = vec_xxpermdi(a1, a2, 0); + D[11] = vec_xxpermdi(a1, a2, 3); + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(AElements, a + 8, lda); + a1 = vec_mergee(AElements[0], AElements[1]); + a2 = vec_mergee(AElements[2], AElements[3]); + D[16] = vec_xxpermdi(a1, a2, 0); + D[18] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(AElements[0], AElements[1]); + a2 = vec_mergeo(AElements[2], AElements[3]); + D[17] = vec_xxpermdi(a1, a2, 0); + D[19] = vec_xxpermdi(a1, a2, 3); + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(AElements, a + 12, lda); + a1 = vec_mergee(AElements[0], AElements[1]); + a2 = vec_mergee(AElements[2], AElements[3]); + D[24] = vec_xxpermdi(a1, a2, 0); + D[26] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(AElements[0], AElements[1]); + a2 = vec_mergeo(AElements[2], AElements[3]); + D[25] = vec_xxpermdi(a1, a2, 0); + D[27] = vec_xxpermdi(a1, a2, 3); + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(A2Elements, a + (lda * 4), lda); + a1 = vec_mergee(A2Elements[0], A2Elements[1]); + a2 = vec_mergee(A2Elements[2], A2Elements[3]); + D[4] = vec_xxpermdi(a1, a2, 0); + D[6] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(A2Elements[0], A2Elements[1]); + a2 = vec_mergeo(A2Elements[2], A2Elements[3]); + D[5] = vec_xxpermdi(a1, a2, 0); + D[7] = vec_xxpermdi(a1, a2, 3); + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(A2Elements, (a + 4) + (lda * 4), lda); + a1 = vec_mergee(A2Elements[0], A2Elements[1]); + a2 = vec_mergee(A2Elements[2], A2Elements[3]); + D[12] = vec_xxpermdi(a1, a2, 0); + D[14] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(A2Elements[0], A2Elements[1]); + a2 = vec_mergeo(A2Elements[2], A2Elements[3]); + D[13] = vec_xxpermdi(a1, a2, 0); + D[15] = vec_xxpermdi(a1, a2, 3); + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(A2Elements, (a + 8) + (lda * 4), lda); + a1 = vec_mergee(A2Elements[0], A2Elements[1]); + a2 = vec_mergee(A2Elements[2], A2Elements[3]); + D[20] = vec_xxpermdi(a1, a2, 0); + D[22] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(A2Elements[0], A2Elements[1]); + a2 = vec_mergeo(A2Elements[2], A2Elements[3]); + D[21] = vec_xxpermdi(a1, a2, 0); + D[23] = vec_xxpermdi(a1, a2, 3); + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(A2Elements, (a + 12) + (lda * 4), lda); + a1 = vec_mergee(A2Elements[0], A2Elements[1]); + a2 = vec_mergee(A2Elements[2], A2Elements[3]); + D[28] = vec_xxpermdi(a1, a2, 0); + D[30] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(A2Elements[0], A2Elements[1]); + a2 = vec_mergeo(A2Elements[2], A2Elements[3]); + D[29] = vec_xxpermdi(a1, a2, 0); + D[31] = vec_xxpermdi(a1, a2, 3); + D += 32; + } else { + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(AElements, a + 4, lda); + a1 = vec_mergee(AElements[0], AElements[1]); + a2 = vec_mergee(AElements[2], AElements[3]); + D[4] = vec_xxpermdi(a1, a2, 0); + D[6] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(AElements[0], AElements[1]); + a2 = vec_mergeo(AElements[2], AElements[3]); + D[5] = vec_xxpermdi(a1, a2, 0); + D[7] = vec_xxpermdi(a1, a2, 3); + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(AElements, a + 8, lda); + a1 = vec_mergee(AElements[0], AElements[1]); + a2 = vec_mergee(AElements[2], AElements[3]); + D[8] = vec_xxpermdi(a1, a2, 0); + D[10] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(AElements[0], AElements[1]); + a2 = vec_mergeo(AElements[2], AElements[3]); + D[9] = vec_xxpermdi(a1, a2, 0); + D[11] = vec_xxpermdi(a1, a2, 3); + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(AElements, a + 12, lda); + a1 = vec_mergee(AElements[0], AElements[1]); + a2 = vec_mergee(AElements[2], AElements[3]); + D[12] = vec_xxpermdi(a1, a2, 0); + D[14] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(AElements[0], AElements[1]); + a2 = vec_mergeo(AElements[2], AElements[3]); + D[13] = vec_xxpermdi(a1, a2, 0); + D[15] = vec_xxpermdi(a1, a2, 3); + D += 16; + } + k -= 16; + a += 16; + } + + while (k >= 8) { + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(AElements, a, lda); + a1 = vec_mergee(AElements[0], AElements[1]); + a2 = vec_mergee(AElements[2], AElements[3]); + D[0] = vec_xxpermdi(a1, a2, 0); + D[2] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(AElements[0], AElements[1]); + a2 = vec_mergeo(AElements[2], AElements[3]); + D[1] = vec_xxpermdi(a1, a2, 0); + D[3] = vec_xxpermdi(a1, a2, 3); + if (RowCount == 8) { + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(AElements, a + 4, lda); + a1 = vec_mergee(AElements[0], AElements[1]); + a2 = vec_mergee(AElements[2], AElements[3]); + D[8] = vec_xxpermdi(a1, a2, 0); + D[10] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(AElements[0], AElements[1]); + a2 = vec_mergeo(AElements[2], AElements[3]); + D[9] = vec_xxpermdi(a1, a2, 0); + D[11] = vec_xxpermdi(a1, a2, 3); + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(A2Elements, a + (lda * 4), lda); + a1 = vec_mergee(A2Elements[0], A2Elements[1]); + a2 = vec_mergee(A2Elements[2], A2Elements[3]); + D[4] = vec_xxpermdi(a1, a2, 0); + D[6] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(A2Elements[0], A2Elements[1]); + a2 = vec_mergeo(A2Elements[2], A2Elements[3]); + D[5] = vec_xxpermdi(a1, a2, 0); + D[7] = vec_xxpermdi(a1, a2, 3); + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(A2Elements, (a + 4) + (lda * 4), lda); + a1 = vec_mergee(A2Elements[0], A2Elements[1]); + a2 = vec_mergee(A2Elements[2], A2Elements[3]); + D[12] = vec_xxpermdi(a1, a2, 0); + D[14] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(A2Elements[0], A2Elements[1]); + a2 = vec_mergeo(A2Elements[2], A2Elements[3]); + D[13] = vec_xxpermdi(a1, a2, 0); + D[15] = vec_xxpermdi(a1, a2, 3); + D += 16; + } else { + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(AElements, a + 4, lda); + a1 = vec_mergee(AElements[0], AElements[1]); + a2 = vec_mergee(AElements[2], AElements[3]); + D[4] = vec_xxpermdi(a1, a2, 0); + D[6] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(AElements[0], AElements[1]); + a2 = vec_mergeo(AElements[2], AElements[3]); + D[5] = vec_xxpermdi(a1, a2, 0); + D[7] = vec_xxpermdi(a1, a2, 3); + D += 8; + } + a += 8; + k -= 8; + } + + while (k >= 4) { + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(AElements, a, lda); + a1 = vec_mergee(AElements[0], AElements[1]); + a2 = vec_mergee(AElements[2], AElements[3]); + D[0] = vec_xxpermdi(a1, a2, 0); + D[2] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(AElements[0], AElements[1]); + a2 = vec_mergeo(AElements[2], AElements[3]); + D[1] = vec_xxpermdi(a1, a2, 0); + D[3] = vec_xxpermdi(a1, a2, 3); + if (RowCount == 8) { + MlasLoopUnroll<4, MlasFgemmLoadAElements>()(A2Elements, a + (lda * 4), lda); + a1 = vec_mergee(A2Elements[0], A2Elements[1]); + a2 = vec_mergee(A2Elements[2], A2Elements[3]); + D[4] = vec_xxpermdi(a1, a2, 0); + D[6] = vec_xxpermdi(a1, a2, 3); + a1 = vec_mergeo(A2Elements[0], A2Elements[1]); + a2 = vec_mergeo(A2Elements[2], A2Elements[3]); + D[5] = vec_xxpermdi(a1, a2, 0); + D[7] = vec_xxpermdi(a1, a2, 3); + D += 8; + } else + D += 4; + a += 4; + k -= 4; + } + + /* When k is less than 4, copy a single element from each row. */ + while (k > 0) { + MlasLoopUnroll<4, MlasSgemmBroadcastAElementsMMA>()(AElements, a, lda); + D[0] = AElements[0]; + if (RowCount == 8) { + MlasLoopUnroll<4, MlasSgemmBroadcastAElementsMMA>()(A2Elements, a + (lda * 4), lda); + D[1] = A2Elements[0]; + D += 2; + } else { + D += 1; + } + a += 1; + k -= 1; + } +} + size_t MLASCALL MlasSgemmKernelPOWER10( @@ -396,17 +658,40 @@ Return Value: --*/ { size_t RowsHandled; + size_t index = CountK * 2; + + MLAS_FLOAT32X4* PackA = + reinterpret_cast(alloca(sizeof(MLAS_FLOAT32X4) * index)); MLAS_FLOAT32X4 AlphaBroadcast = MlasBroadcastFloat32x4(alpha); if (CountM >= 8) { - RowsHandled = MlasSgemmMMAProcessCount<4>(A, B, C, 8 ,CountK, CountN, lda, ldc, AlphaBroadcast, ZeroMode); +#ifdef _AIX + MlasSgemmPackA<8>(PackA, A, lda, CountK); +#else + if (CountK >= 16 && !(CountK % 16)) { + PackAKernelPOWER10(PackA, A, lda, CountK, 8); + } else { + MlasSgemmPackA<8>(PackA, A, lda, CountK); + } +#endif + + RowsHandled = MlasSgemmMMAProcessCount<4>(PackA, B, C, 8, CountK, CountN, ldc, AlphaBroadcast, ZeroMode); } else if (CountM >= 4) { - RowsHandled = MlasSgemmMMAProcessCount<4>(A, B, C, 4, CountK, CountN, lda, ldc, AlphaBroadcast, ZeroMode); + memset(PackA + CountK, 0, sizeof(MLAS_FLOAT32X4) * CountK); +#ifdef _AIX + MlasSgemmPackA<4>(PackA, A, lda, CountK); +#else + if (CountK >= 16 && !(CountK % 16)) { + PackAKernelPOWER10(PackA, A, lda, CountK, 4); + } else { + MlasSgemmPackA<4>(PackA, A, lda, CountK); + } +#endif + RowsHandled = MlasSgemmMMAProcessCount<4>(PackA, B, C, 4, CountK, CountN, ldc, AlphaBroadcast, ZeroMode); } else if (CountM >= 2) { RowsHandled = MlasSgemmProcessCount<2>(A, B, C, CountK, CountN, lda, ldc, AlphaBroadcast, ZeroMode); } else { RowsHandled = MlasSgemmProcessCount<1>(A, B, C, CountK, CountN, lda, ldc, AlphaBroadcast, ZeroMode); } - return RowsHandled; } diff --git a/onnxruntime/core/mlas/lib/power/SgemmKernelPackA.S b/onnxruntime/core/mlas/lib/power/SgemmKernelPackA.S new file mode 100644 index 0000000000000..fa2d79ea6deff --- /dev/null +++ b/onnxruntime/core/mlas/lib/power/SgemmKernelPackA.S @@ -0,0 +1,247 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SgemmKernelPackA.S + +Abstract: + + This module implements the POWER10 kernel for packing matrix A for single precision SGEMM. + + This implementation targets power10 using VSX instructions. + +--*/ +/*++ +Routine Description: + + This routine is an inner kernel to pack matrix A for rows 4 or 8. + +Arguments: + + D (r3) - Supplies the address of Packed A. + + A (r4) - Supplies the address of matrix A. + + lda (r5) - LDA. + + k (r6) - Supplies the number of columns from matrix A. + + RowCount (r7) - Supplies the number of rows to process. + + +Return Value: + + None. +--*/ +#include "asmmacro.h" +.text +FUNCTION_ENTRY PackAKernelPOWER10 + slwi 9,5,2 + cmpldi 7,8 + add 8,4,9 + add 10,8,9 + add 11,10,9 + dcbt 0,4 + dcbt 0,8 + dcbt 0,10 + dcbt 0,11 + blt L_loop +L_Rows8: + lxvp 32,0(4) + lxvp 42,32(4) + addi 4,4,64 + dcbt 0,4 + lxvp 34,0(8) //a+lda + lxvp 44,32(8) //a+32+lda + lxvp 36,0(10) //a+2*lda + lxvp 46,32(10) //a+32+2*lda + lxvp 38,0(11) //a+3*lda + lxvp 48,32(11) //a+32+3*lda + add 7,11,9 + dcbt 0,7 + add 8,7,9 + dcbt 0,8 + add 10,8,9 + dcbt 0,10 + add 11,10,9 + dcbt 0,11 + vmrgow 8,3,1 + vmrgew 18,3,1 + vmrgow 9,7,5 + vmrgew 19,7,5 + xxpermdi 1,41,40,3 + xxpermdi 0,51,50,3 + xxpermdi 3,41,40,0 + xxpermdi 2,51,50,0 + vmrgow 8,2,0 + vmrgow 9,6,4 + vmrgew 18,2,0 + vmrgew 19,6,4 + stxvp 0,0(3) + xxpermdi 9,41,40,3 + xxpermdi 8,51,50,3 + xxpermdi 11,41,40,0 + xxpermdi 10,51,50,0 + stxvp 2,32(3) + stxvp 8,128(3) + stxvp 10,160(3) + + vmrgow 0,13,11 + vmrgow 1,17,15 + vmrgew 18,13,11 + vmrgew 19,17,15 + xxpermdi 5,33,32,3 + xxpermdi 7,33,32,0 + xxpermdi 4,51,50,3 + xxpermdi 6,51,50,0 + vmrgow 0,12,10 + vmrgow 1,16,14 + stxvp 4,256(3) + stxvp 6,288(3) + vmrgew 18,12,10 + vmrgew 19,16,14 + xxpermdi 9,33,32,3 + xxpermdi 8,51,50,3 + xxpermdi 11,33,32,0 + xxpermdi 10,51,50,0 + lxvp 32,0(7) //a+4*lda + lxvp 34,0(8) //a+5*lda + lxvp 36,0(10) //a+6*lda + lxvp 38, 0(11) //a+7*lda + + stxvp 8,384(3) + stxvp 10,416(3) + lxvp 42,32(7) //a+32+4*lda + + vmrgow 8,3,1 + vmrgew 18,3,1 + vmrgow 9,7,5 + vmrgew 19,7,5 + lxvp 44,32(8) //a+32+5*lda + lxvp 46,32(10) //a+32+6*lda + + xxpermdi 1,41,40,3 + xxpermdi 0,51,50,3 + xxpermdi 3,41,40,0 + xxpermdi 2,51,50,0 + lxvp 48,32(11) //a+32+7*lda + add 8,4,9 + dcbt 0,8 + add 10,8,9 + dcbt 0,10 + add 11,10,9 + dcbt 0,11 + stxvp 0,64(3) + stxvp 2,96(3) + vmrgow 8,2,0 + vmrgow 9,6,4 + vmrgew 18,2,0 + vmrgew 19,6,4 + vmrgow 0,13,11 + vmrgow 1,17,15 + + xxpermdi 9,41,40,3 + xxpermdi 8,51,50,3 + xxpermdi 11,41,40,0 + xxpermdi 10,51,50,0 + vmrgew 18,13,11 + vmrgew 19,17,15 + stxvp 8,192(3) + stxvp 10,224(3) + xxpermdi 5,33,32,3 + xxpermdi 4,51,50,3 + xxpermdi 7,33,32,0 + xxpermdi 6,51,50,0 + vmrgow 0,12,10 + vmrgow 1,16,14 + stxvp 4,320(3) + stxvp 6,352(3) + vmrgew 18,12,10 + vmrgew 19,16,14 + xxpermdi 9,33,32,3 + xxpermdi 8,51,50,3 + xxpermdi 11,33,32,0 + xxpermdi 10,51,50,0 + + stxvp 8,448(3) + stxvp 10,480(3) + addi 6,6,-16 + cmpldi 6,16 + + addi 3,3,512 + bge L_Rows8 + b L_exit + +L_loop: + lxvp 32,0(4) + lxvp 42,32(4) + addi 4,4,64 + dcbt 0,4 + lxvp 34,0(8) //a+lda + lxvp 44,32(8) //a+32+lda + lxvp 36,0(10) //a+2*lda + lxvp 46,32(10) //a+32+2*lda + lxvp 38,0(11) //a+3*lda + lxvp 48,32(11) //a+32+3*lda + vmrgow 8,3,1 + vmrgew 18,3,1 + vmrgow 9,7,5 + vmrgew 19,7,5 + + add 8,4,9 + dcbt 0,8 + add 10,8,9 + dcbt 0,10 + add 11,10,9 + dcbt 0,11 + + xxpermdi 1,41,40,3 + xxpermdi 0,51,50,3 + xxpermdi 3,41,40,0 + xxpermdi 2,51,50,0 + vmrgow 8,2,0 + vmrgow 9,6,4 + vmrgew 18,2,0 + vmrgew 19,6,4 + stxvp 0,0(3) + + xxpermdi 9,41,40,3 + xxpermdi 8,51,50,3 + xxpermdi 11,41,40,0 + xxpermdi 10,51,50,0 + stxvp 2,32(3) + stxvp 8,64(3) + stxvp 10,96(3) + + vmrgow 0,13,11 + vmrgow 1,17,15 + vmrgew 18,13,11 + vmrgew 19,17,15 + xxpermdi 5,33,32,3 + xxpermdi 7,33,32,0 + xxpermdi 4,51,50,3 + xxpermdi 6,51,50,0 + vmrgow 0,12,10 + vmrgow 1,16,14 + stxvp 4,128(3) + stxvp 6,160(3) + vmrgew 18,12,10 + vmrgew 19,16,14 + xxpermdi 9,33,32,3 + xxpermdi 8,51,50,3 + xxpermdi 11,33,32,0 + xxpermdi 10,51,50,0 + stxvp 8,192(3) + stxvp 10,224(3) + + addi 3,3,256 + addi 6,6,-16 + cmpldi 6,16 + bge L_loop + +L_exit: + blr diff --git a/onnxruntime/core/mlas/lib/power/asmmacro.h b/onnxruntime/core/mlas/lib/power/asmmacro.h new file mode 100644 index 0000000000000..18b39b5b32804 --- /dev/null +++ b/onnxruntime/core/mlas/lib/power/asmmacro.h @@ -0,0 +1,47 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + asmmacro.h + +Abstract: + + This module implements common macros for the assembly modules. + +--*/ + +#if defined(__APPLE__) +#define C_UNDERSCORE(symbol) _##symbol +#else +#define C_UNDERSCORE(symbol) symbol +#endif + +/*++ + +Macro Description: + + This macro emits the assembler directives to annotate a new function. + +Arguments: + + FunctionName - Supplies the name of the function. + +--*/ + + .macro FUNCTION_ENTRY FunctionName + + .p2align 4 +#if defined(__APPLE__) + .globl _\FunctionName\() +_\FunctionName\(): +#else + .globl \FunctionName\() + .type \FunctionName\(),@function +\FunctionName\(): +#endif + + .endm