From 848b6f6c280c4ee3ca856532752a1408770f4993 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Tue, 4 Jun 2024 20:00:03 +0000 Subject: [PATCH 01/36] Add tuning parameters with updated doc comments --- tensilelite/Tensile/Common.py | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index 7e5a74d16f..807829282e 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -869,6 +869,49 @@ def getArchitectureName(gfxName): # Only support for kernel whose totalVgpr counts less than 256 and gcn that has control bit ACC_CD. "MIArchVgpr": [False, True], + # StreamK kernels divide work evenly among CUs by splitting along MT and K dimensions + # Total work units are calculated as (#MTs x #LoopIters) and divided among workgroups + # In most cases each workgroup will calculate a partial tile that are accumulated in a fixup step in the same kernel + # 0: Standard data-parallel kernel + # 1: Basic StreamK + # 2: Two-Tile StreamK (each WG completes an even number of sk iterations, followed by an even number of dp tiles) + # 3: Two-Tile StreamK with DP before SK tiles + # StreamK kernels can adjust the number of CUs being used. + # Using fewer sometimes increases overall throughput by allowing other kernels to run in parallel. + # StreamK grid is controlled by setting these enviornment variables: + # TENSILE_STREAMK_DYNAMIC_GRID enables dynamic grid mode, which automatically limits the number of CUs used for small + # problems to a subset based on the number of output tiles. + # 0 = off (default) + # 1 = on + # 2 = also reduce CUs used for large sizes to improve data-parallel portion and reduce power + # 3 = analytical grid size predictor + # TENSILE_STREAMK_MAX_CUS allows the user to manually set maximum number of CUs used, which could free up some CUs for + # other operations to run in parallel with gemm. + # 0 = use all CUs (default) + # TENSILE_STREAMK_GRID_MULTIPLIER lets you set how many workgroups are created per CU being used. + # 1 = 1 WG per CU (default) + # TENSILE_STREAMK_FIXED_GRID lets you override the default grid size with a specific number + # 0 = override disabled (default) + "StreamK": [0, 1, 2, 3], + # Determines if StreamK kernel uses atomics + # 0: uses workspace to store partial tiles, accumulate in deterministic fix-up step + # 1: uses atomics to accumulate partial tiles + "StreamKAtomic": [0, 1], + # Enables XCC-based remapping of workgroups, set the value to the number of XCCs + # for the device/configuration being used + # 0: uses default workgroup assignment + # 2+: remaps workgroups to be contiguous within an XCC for a given number of XCCs + "StreamKXCCMapping": [0] + list(range(2, 9)), + # Debug settings for stream-k kernels to disable parts of the kernel + # Bit 0: Don't generate fixup code + # Bit 1: Don't generate write to partials code + # Both parts can be disabled together + # 0 = Debug mode off, generate full kernel + # 1 = No fixup + # 2 = No partials + # 3 = Nofixup and no partials + "DebugStreamK": [0, 1, 2, 3], + # Controls desired width (#elements) for loads from global memory -> LDS. # and eliminates the pointer unshift logic # -1 : Set GlobalReadVectorWidth = VectorWidth @@ -1119,6 +1162,10 @@ def getArchitectureName(gfxName): {"StoreSyncOpt": [ 0 ] }, {"GroupLoadStore": [ False ] }, {"MIArchVgpr": [ False ] }, + {"StreamK": [ 0 ] }, + {"StreamKAtomic": [ 0 ] }, + {"StreamKXCCMapping": [ 0 ] }, + {"DebugStreamK": [ 0 ] }, {"ActivationFused": [ True ] }, {"ActivationFuncCall": [ True ] }, {"ActivationAlt": [ False ] }, From 6ff49e1dbc00a9be9658c8fbefc3c25218182e35 Mon Sep 17 00:00:00 2001 From: Bryant Nelson Date: Wed, 5 Jun 2024 10:54:17 -0500 Subject: [PATCH 02/36] SK Port: (Host) Debug settings and Grid Selector (#1) --- .../Source/lib/include/Tensile/AMDGPU.hpp | 40 +++++++++++++++++ .../include/Tensile/ContractionSolution.hpp | 3 ++ .../Tensile/Source/lib/source/AMDGPU.cpp | 5 +++ .../Source/lib/source/ContractionSolution.cpp | 45 +++++++++++++++++++ 4 files changed, 93 insertions(+) diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPU.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPU.hpp index 2c45273af6..2650dd641f 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPU.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/AMDGPU.hpp @@ -184,6 +184,11 @@ namespace Tensile int wavefrontSize = 64; int simdPerCu = 4; int computeUnitCount = 0; + int skDynamicGrid = 0; + int skMaxCUs = 0; + int skGridMultiplier = 1; + int skFixedGrid = 0; + int skFullTiles = 0; std::string deviceName; virtual bool runsKernelTargeting(Processor p) const; @@ -199,6 +204,41 @@ namespace Tensile virtual std::string description() const; + const int getSKDynamicGrid() const + { + static const char* envStr = std::getenv("TENSILE_STREAMK_DYNAMIC_GRID"); + static const int value = (envStr == NULL ? 0 : std::atoi(envStr)); + return value; + } + + const int getSKMaxCUs() const + { + static const char* envStr = std::getenv("TENSILE_STREAMK_MAX_CUS"); + static const int value = (envStr == NULL ? 0 : std::atoi(envStr)); + return value; + } + + const int getSKGridMultiplier() const + { + static const char* envStr = std::getenv("TENSILE_STREAMK_GRID_MULTIPLIER"); + static const int value = (envStr == NULL ? 1 : std::atoi(envStr)); + return value; + } + + const int getSKFixedGrid() const + { + static const char* envStr = std::getenv("TENSILE_STREAMK_FIXED_GRID"); + static const int value = (envStr == NULL ? 0 : std::atoi(envStr)); + return value; + } + + const int getSKFullTiles() const + { + static const char* envStr = std::getenv("TENSILE_STREAMK_FULL_TILES"); + static const int value = (envStr == NULL ? 0 : std::atoi(envStr)); + return value; + } + bool operator==(AMDGPU const& rhs) const { return processor == rhs.processor && computeUnitCount == rhs.computeUnitCount; diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp index da6fde32c3..42290797e1 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp @@ -224,6 +224,9 @@ namespace Tensile size_t requiredWorkspaceSizeGroupedGemm(std::vector const& problems) const; size_t requiredHostSizeGroupedGemmSingle(Problem const& problem) const; + size_t getSKGrid(Hardware const& hardware, size_t tiles) const; + size_t partialTileSize(size_t skGrid) const; + static float computeGranularity(float x); Granularities computeGranularities( diff --git a/tensilelite/Tensile/Source/lib/source/AMDGPU.cpp b/tensilelite/Tensile/Source/lib/source/AMDGPU.cpp index 9f5d03dcfa..f2916e05a2 100644 --- a/tensilelite/Tensile/Source/lib/source/AMDGPU.cpp +++ b/tensilelite/Tensile/Source/lib/source/AMDGPU.cpp @@ -39,6 +39,11 @@ namespace Tensile : processor(p) , computeUnitCount(cus) , deviceName(name) + , skDynamicGrid(getSKDynamicGrid()) + , skMaxCUs(getSKMaxCUs()) + , skGridMultiplier(getSKGridMultiplier()) + , skFixedGrid(getSKFixedGrid()) + , skFullTiles(getSKFullTiles()) { } diff --git a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp index 4c558c0302..4f1fa80591 100644 --- a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp +++ b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp @@ -2602,6 +2602,51 @@ namespace Tensile return h_args.size(); } + size_t ContractionSolution::getSKGrid(Hardware const& hardware, size_t tiles) const + { + AMDGPU const* pAMDGPU = dynamic_cast(&hardware); + if(pAMDGPU->skFixedGrid > 0) + return pAMDGPU->skFixedGrid; + assert(pAMDGPU != nullptr && pAMDGPU->computeUnitCount != 0); + size_t cuCount = pAMDGPU->computeUnitCount; + size_t skGrid = cuCount; + if(pAMDGPU->skDynamicGrid == 2 && tiles > skGrid) + { + for(size_t i = 1; i <= 32; i *= 2) + { + size_t tilesPerCU = CeilDivide(i * tiles, skGrid); + size_t reducedGrid = CeilDivide(i * tiles, tilesPerCU); + float utilization = ((float)reducedGrid) / ((float)skGrid); + if(utilization > 0.75f) + { + if(utilization < 1.0f) + skGrid = reducedGrid; + break; + } + } + } + if(pAMDGPU->skMaxCUs > 0) + skGrid = min(skGrid, pAMDGPU->skMaxCUs); + if(pAMDGPU->skDynamicGrid) + skGrid = min(skGrid, tiles); + if(pAMDGPU->skGridMultiplier > 1) + skGrid = skGrid * pAMDGPU->skGridMultiplier; + return skGrid; + } + + size_t ContractionSolution::partialTileSize(size_t skGrid) const + { + size_t size = 0; + + size_t tileSize + = sizeMapping.macroTile.x * sizeMapping.macroTile.y * sizeMapping.workspaceSizePerElemC; + size += tileSize * skGrid; // Partials tile per WG + // TODO batches + // TODO round up for alignment? + + return size; + } + float ContractionSolution::computeGranularity(float x) { return x / ceil(x); From f25ba0bdb04e400d50792b476d790c92e5327e20 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Thu, 6 Jun 2024 15:35:36 +0000 Subject: [PATCH 03/36] Stream-K kernel arguments, metadata, and launch grid code --- tensilelite/Tensile/Components/Signature.py | 21 +++ tensilelite/Tensile/Contractions.py | 12 +- tensilelite/Tensile/KernelWriter.py | 40 +++++- .../include/Tensile/ContractionProblem.hpp | 3 + .../include/Tensile/ContractionSolution.hpp | 88 +++++++------ .../Tensile/ContractionSolution_fwd.hpp | 1 + .../Serialization/ContractionSolution.hpp | 6 +- .../Source/lib/source/ContractionProblem.cpp | 46 +++++++ .../Source/lib/source/ContractionSolution.cpp | 122 ++++++++++++++++-- 9 files changed, 279 insertions(+), 60 deletions(-) diff --git a/tensilelite/Tensile/Components/Signature.py b/tensilelite/Tensile/Components/Signature.py index d0bcb53d97..95e93ab0c2 100644 --- a/tensilelite/Tensile/Components/Signature.py +++ b/tensilelite/Tensile/Components/Signature.py @@ -160,6 +160,10 @@ def __call__(self, writer) -> SignatureBase: if kernel["ProblemType"]["Sparse"]: signature.addArg("MetaData", SVK.SIG_GLOBALBUFFER, "void" , "generic") + if kernel["StreamK"] > 0 and kernel["StreamKAtomic"] == 0: + signature.addArg("AddressWS", SVK.SIG_GLOBALBUFFER, dstValueType, "generic") + signature.addArg("AddressFlags", SVK.SIG_GLOBALBUFFER, dstValueType, "generic") + for i in range(0, writer.states.d.numSgprStrides): signature.addArg( "strideD%u"%i, SVK.SIG_VALUE, "u32") userArgumentsInfo.gemmArgumentSize += 4 @@ -239,6 +243,23 @@ def __call__(self, writer) -> SignatureBase: signature.addArg( "Synchronizer", SVK.SIG_GLOBALBUFFER, cptValueType, "generic") signature.addArg( "GSUSync", SVK.SIG_VALUE, "u32") + if kernel["StreamK"]: + # StreamK args + signature.addArg("MagicNumberProblemNumGroupTiles0", SVK.SIG_VALUE, "u32") + signature.addArg("MagicShiftProblemNumGroupTiles0", SVK.SIG_VALUE, "u32") + signature.addArg("ItersPerTile", SVK.SIG_VALUE, "u32") + signature.addArg("MagicNumberItersPerTile", SVK.SIG_VALUE, "u32") + signature.addArg("MagicShiftItersPerTile", SVK.SIG_VALUE, "u32") + signature.addArg("MagicNumProblemNumGroupTiles0By1", SVK.SIG_VALUE, "u32") + signature.addArg("MagicShiftProblemNumGroupTiles0By1", SVK.SIG_VALUE, "u32") + signature.addArg("TotalIters", SVK.SIG_VALUE, "u32") + signature.addArg("SKItersPerWG", SVK.SIG_VALUE, "u32") + if kernel["StreamK"] >= 2: # Two-tile SK + signature.addArg("skGrid", SVK.SIG_VALUE, "u32") + signature.addArg("skTiles", SVK.SIG_VALUE, "u32") + signature.addArg("skExtraIters", SVK.SIG_VALUE, "u32") + # "dpTilesPerWG" + activationType = ActivationType("all") for name in activationType.getAdditionalArgStringList(): userArgumentsInfo.activationSize += userArgumentsInfo.actMaxSize diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index 1ddd32c6da..e20320b918 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -445,6 +445,10 @@ def CompoundPredicates(cls, state, problemType): if ('GlobalSplitU' in state) and (state['GlobalSplitU'] > 1): if ('_GlobalAccumulation' not in state) or (state['_GlobalAccumulation'] != 'MultipleBuffer'): rv += [cls("DeterministicMode", value = False)] + + if ('StreamK' in state) and (state['StreamK'] > 0) and ('StreamKAtomic' in state) and (state['StreamKAtomic'] == 1): + # StreamKAtomic = 1 uses atomic for partial tiles + rv += [cls("DeterministicMode", value = False)] # if bufferload is performed, we output some predication info for host side, # to prevent from some extremely large problems from launching and causing bufferload offset limit < 2^32 @@ -474,7 +478,7 @@ def CompoundPredicates(cls, state, problemType): if 'BufferStore' in state and state['BufferStore'] == True: rv += [cls('BufferStoreOffsetLimitCheck', value=state['MacroTile1'])] - if '_GlobalAccumulation' in state and state['_GlobalAccumulation'] != None: + if '_GlobalAccumulation' in state and state['_GlobalAccumulation'] != None and not state["StreamK"]: value = globalParameters['MinKForGSU'] rv += [cls('GlobalSplitUCheckMinK', value=[value, state["GlobalSplitU"]])] @@ -508,6 +512,8 @@ class SizeMapping: 'workGroupMapping', 'packBatchDims', 'magicDivAlg', + 'streamK', + 'streamKAtomic', 'sourceKernel', 'globalAccumulation', 'workspaceSizePerElemC', @@ -526,6 +532,8 @@ def FromOriginalState(cls, d): globalAccum = 2 if d['_GlobalAccumulation'] == 'MultipleBufferSingleKernel': globalAccum = 3 + if d['_GlobalAccumulation'] == 'PartialsBuffer': + globalAccum = 4 return cls(waveNum = d['NumThreads'] // d['WavefrontSize'], workGroup = d['WorkGroup'], macroTile = cls.ReadOriginalMacroTile(d), @@ -543,6 +551,8 @@ def FromOriginalState(cls, d): globalSplitU = d['GlobalSplitU'], staggerStrideShift = d['_staggerStrideShift'] if '_staggerStrideShift' in d else 0, packBatchDims = 0, + streamK = d['StreamK'] if 'StreamK' in d else 0, + streamKAtomic = d['StreamKAtomic'] if 'StreamKAtomic' in d else 0, magicDivAlg = d.get('MagicDivAlg', 1), sourceKernel = d['KernelLanguage'] == 'Source', globalAccumulation = globalAccum, diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 922c98306e..345471a7f2 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -227,13 +227,14 @@ class StateValues: numStoreSgprNames: List[str] = field(init=False) # For post-loop kernel args numStoreSgprNameSizes: List[int] = field(init=False) # For post-loop kernel args numStoreSgprToLoad: int = 0 # For post-loop kernel args - numStoreSgprNames2: List[str] = field(init=False) # For post-loop kernel args - numStoreSgprNameSizes2: List[int] = field(init=False) # For post-loop kernel args - numStoreSgprToLoad2: int = 0 # For post-loop kernel args + numStoreSgprNames2: List[str] = field(init=False) # For post-loop kernel args + numStoreSgprNameSizes2: List[int] = field(init=False) # For post-loop kernel args + numStoreSgprToLoad2: int = 0 # For post-loop kernel args numStoreSgprInst: int = 0 # For pose-loop kernel args numStoreSgprInstExt: int = 0 # For pose-loop kernel args numSgprAddressBias: int = 0 numSgprAddressGSUSync: int = 0 + numSgprStreamK: int = 0 BiasType: int = 0 BiasStride: int = 0 BiasDim: int = 0 @@ -3479,6 +3480,8 @@ def readWriteVectors(mat, vw, kernel): numSgprAddressC = self.states.rpga # til end numSgprAddressA = self.states.rpga # til read offsets numSgprAddressB = self.states.rpga # til read offsets + numSgprAddressWS = self.states.rpga + numSgprAddressFlags = self.states.rpga numSgprAddressMetadata = self.states.rpga if kernel["ProblemType"]["Sparse"] else 0 @@ -3634,13 +3637,18 @@ def readWriteVectors(mat, vw, kernel): self.defineSgpr("AddressB", numSgprAddressB) if kernel["ProblemType"]["Sparse"]: self.defineSgpr("AddressMetadata", numSgprAddressMetadata) + if kernel["StreamK"] > 0 and kernel["StreamKAtomic"] == 0: + self.defineSgpr("AddressWS", numSgprAddressWS) + self.defineSgpr("AddressFlags", numSgprAddressFlags) + self.states.numSgprStreamK += numSgprAddressWS + numSgprAddressFlags + #asm input interface depen self.defineSgpr("StridesD", self.states.d.numSgprStrides) self.defineSgpr("StridesC", self.states.c.numSgprStrides) self.defineSgpr("StridesA", self.states.a.numSgprStrides) self.defineSgpr("StridesB", self.states.b.numSgprStrides) if kernel["ProblemType"]["Sparse"]: - self.defineSgpr("StridesMetadata", self.states.m.numSgprStrides) + self.defineSgpr("StridesMetadata", self.states.m.numSgprStrides) # for packed batches without stride restrictions need to do something different here assert sorted(kernel["PackedC0IdxChars"]+kernel["PackedC1IdxChars"]) == \ @@ -3662,8 +3670,6 @@ def readWriteVectors(mat, vw, kernel): self.defineSgpr("Beta", numSgprBeta, numSgprBeta) self.states.numSgprBeta = numSgprBeta - self.defineSgpr("GSU", 1) # Can't move to the front because of the preload arguments - if GSUAMBSK: self.defineSgpr("GSUSync", 1) self.states.numSgprAddressGSUSync += 1 @@ -3672,6 +3678,27 @@ def readWriteVectors(mat, vw, kernel): self.defineSgpr("Synchronizer", 2) self.states.numSgprAddressGSUSync += 2 + if kernel["StreamK"]: + # StreamK args + self.defineSgpr("MagicNumberProblemNumGroupTiles0", 1) # Magic number to use for division + self.defineSgpr("MagicShiftProblemNumGroupTiles0", 1) # Magic shift/abit to use for division alg 2 + self.defineSgpr("ItersPerTile", 1) + self.defineSgpr("MagicNumberItersPerTile", 1) + self.defineSgpr("MagicShiftItersPerTile", 1) + self.defineSgpr("MagicNumProblemNumGroupTiles0By1", 1) # for PKAB, use for Magic Div Alg 2 by (nwg0*nwg1) + self.defineSgpr("MagicShiftProblemNumGroupTiles0By1", 1) # for PKAB, use for Magic Div Alg 2 by (nwg0*nwg1) + self.defineSgpr("TotalIters", 1) + self.defineSgpr("SKItersPerWG", 1) + self.states.numSgprStreamK += 9 + if kernel["StreamK"] >= 2: # Two-tile SK + self.defineSgpr("skGrid", 1) + self.defineSgpr("skTiles", 1) + self.defineSgpr("skExtraIters", 1) + # self.defineSgpr("dpTilesPerWG", 1, kernarg=True) + self.states.numSgprStreamK += 3 + + self.defineSgpr("GSU", 1) # Can't move to the front because of the preload arguments + #------------------------ # Registers defined below this point are not available in the post-loop # Post-loop is after tail loop exits, ie the store code. @@ -3708,6 +3735,7 @@ def readWriteVectors(mat, vw, kernel): numSgprAddressD + numSgprAddressC + numSgprAddressA + numSgprAddressB + numSgprAlpha + numSgprAddressMetadata + \ (numSgprBeta if kernel["ProblemType"]["UseBeta"] else 0) + \ self.states.d.numSgprStrides + self.states.c.numSgprStrides + self.states.a.numSgprStrides + self.states.b.numSgprStrides + self.states.m.numSgprStrides + \ + self.states.numSgprStreamK + \ len(kernel["PackedC0IdxChars"][:-1])*2 + len(kernel["PackedC1IdxChars"][:-1])*2 # Get kernel argument end here ################################### diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem.hpp index cd89d0b175..494227f05e 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem.hpp @@ -1016,6 +1016,9 @@ namespace Tensile return getOperationDescription(); } + size_t getNumTiles(SizeMapping const& sizeMapping) const; + size_t getItersPerTile(SizeMapping const& sizeMapping) const; + void checkPersistentKernelEligibility(ContractionSolution const& solution, Hardware const& hardware); diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp index 42290797e1..73d4143ca9 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp @@ -103,6 +103,49 @@ namespace Tensile size_t depthUorMT1; }; + struct SizeMapping + { + size_t waveNum; + + dim3 workGroupSize; + dim3 threadTile; + dim3 macroTile; + + std::array matrixInstruction; + size_t grvwA = 1; + size_t grvwB = 1; + size_t gwvwC = 1; + size_t gwvwD = 1; + + size_t staggerU = 0; + size_t staggerUMapping = 0; + size_t depthU = 0; + size_t globalSplitUPGR = 0; + size_t globalSplitU = 0; + size_t staggerStrideShift = 0; + int workGroupMapping = 0; + + size_t packBatchDims = 0; + int packSummationDims = 0; + int magicDivAlg = 1; + int streamK = 0; + int streamKAtomic = 0; + int persistentKernel = 0; + bool persistentKernelAlongBatch = false; + + bool sourceKernel = false; + + int globalAccumulation = 0; + size_t workspaceSizePerElemC = 0; + size_t workspaceSizePerElemBias = 0; + + bool activationFused = true; + + std::string customKernelName; + + int workGroupMappingXCC = 1; + }; + /** * Represents a single kernel or set of kernels that can perform a single * tensor contraction. @@ -316,6 +359,7 @@ namespace Tensile void singleCallArgs(Problem const& problem, ContractionInputs const& inputs, uint32_t const& workspaceOffsetInByte, + Hardware const* hardware, KA& args) const; // Common kernel related arguments (e.g. gemm_count, arg type, MT, GSU...) @@ -334,7 +378,8 @@ namespace Tensile template KernelInvocation generateSingleCall(Problem const& problem, - ContractionInputs const& inputs) const; + ContractionInputs const& inputs, + Hardware const& hardware) const; template KernelInvocation generateSingleCallGroupedGemm(std::vector const& problems, @@ -403,47 +448,6 @@ namespace Tensile size_t mt1, size_t vw) const; - struct SizeMapping - { - size_t waveNum; - - dim3 workGroupSize; - dim3 threadTile; - dim3 macroTile; - - std::array matrixInstruction; - size_t grvwA = 1; - size_t grvwB = 1; - size_t gwvwC = 1; - size_t gwvwD = 1; - - size_t staggerU = 0; - size_t staggerUMapping = 0; - size_t depthU = 0; - size_t globalSplitUPGR = 0; - size_t globalSplitU = 0; - size_t staggerStrideShift = 0; - int workGroupMapping = 0; - - size_t packBatchDims = 0; - int packSummationDims = 0; - int magicDivAlg = 1; - int persistentKernel = 0; - bool persistentKernelAlongBatch = false; - - bool sourceKernel = false; - - int globalAccumulation = 0; - size_t workspaceSizePerElemC = 0; - size_t workspaceSizePerElemBias = 0; - - bool activationFused = true; - - std::string customKernelName; - - int workGroupMappingXCC = 1; - }; - struct InternalArgsSupport { bool gsu = true; diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution_fwd.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution_fwd.hpp index ea6efba7be..48f23a8250 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution_fwd.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution_fwd.hpp @@ -29,4 +29,5 @@ namespace Tensile { class ContractionSolution; + struct SizeMapping; } diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionSolution.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionSolution.hpp index 9a876f9565..abf1d83e2d 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionSolution.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionSolution.hpp @@ -73,10 +73,10 @@ namespace Tensile }; template - struct MappingTraits + struct MappingTraits { using iot = IOTraits; - static void mapping(IO& io, ContractionSolution::SizeMapping& s) + static void mapping(IO& io, SizeMapping& s) { iot::mapRequired(io, "waveNum", s.waveNum); @@ -100,6 +100,8 @@ namespace Tensile iot::mapOptional(io, "packBatchDims", s.packBatchDims); iot::mapOptional(io, "packSummationDims", s.packSummationDims); iot::mapOptional(io, "magicDivAlg", s.magicDivAlg); + iot::mapOptional(io, "streamK", s.streamK); + iot::mapOptional(io, "streamKAtomic", s.streamKAtomic); iot::mapOptional(io, "persistentKernel", s.persistentKernel); iot::mapOptional(io, "persistentKernelAlongBatch", s.persistentKernelAlongBatch); iot::mapRequired(io, "sourceKernel", s.sourceKernel); diff --git a/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp b/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp index c7c48ff0f5..03e65486a1 100644 --- a/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp +++ b/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp @@ -640,6 +640,52 @@ namespace Tensile return found->i; } + size_t ContractionProblemGemm::getNumTiles(SizeMapping const& sizeMapping) const + { + // Get the normal WorkGroup numbers by sizeMapping MacroTile + dim3 numWG(1, 1, 1); + for(size_t i = 0; i < m_freeIndicesA.size(); i++) + { + numWG.x *= m_freeSizesA.at(i); + } + for(size_t i = 0; i < m_freeIndicesB.size(); i++) + { + numWG.y *= m_freeSizesB.at(i); + } + for(size_t i = 0; i < m_batchIndices.size(); i++) + { + if(sizeMapping.packBatchDims & 0x1) + numWG.x *= m_batchSizes[i]; + if(sizeMapping.packBatchDims & 0x2) + numWG.y *= m_batchSizes[i]; + if(!sizeMapping.packBatchDims) + numWG.z *= m_batchSizes[i]; + } + + numWG.x = CeilDivide(numWG.x, sizeMapping.macroTile.x); + numWG.y = CeilDivide(numWG.y, sizeMapping.macroTile.y); + numWG.y *= sizeMapping.globalSplitU; + + size_t problemTiles = numWG.x * numWG.y; + if(sizeMapping.persistentKernelAlongBatch || sizeMapping.streamK != 0) + problemTiles *= numWG.z; + + return problemTiles; + } + + size_t ContractionProblemGemm::getItersPerTile(SizeMapping const& sizeMapping) const + { + size_t boundSize = 1; + for(size_t i = 0; i < m_boundIndices.size(); ++i) + { + boundSize *= m_boundSizes[i]; + } + + size_t itersPerTile = CeilDivide(boundSize, sizeMapping.depthU); + + return itersPerTile; + } + void ContractionProblemGemm::checkPersistentKernelEligibility( ContractionSolution const& solution, Hardware const& hardware) { diff --git a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp index 4f1fa80591..fbaba6413e 100644 --- a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp +++ b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp @@ -517,8 +517,9 @@ namespace Tensile template void ContractionSolution::singleCallArgs(ContractionSolution::Problem const& problem, ContractionInputs const& inputs, - uint32_t const& workspaceOffsetInByte, - KA& args) const + uint32_t const& workspaceOffsetInByte, + Hardware const* hardware, + KA& args) const { if(debugKernel) { @@ -550,7 +551,7 @@ namespace Tensile && (problemType.computeType != problemType.dType || problemType.activationType != ActivationType::None)) singleWSD = true; - if(gsu > 1 + if(gsu > 1 && sizeMapping.streamK == 0 && ((singleWSD || sizeMapping.globalAccumulation == 2) || (sizeMapping.globalAccumulation == 3 ))) { @@ -590,10 +591,27 @@ namespace Tensile if(problemType.sparse) args.template append("metadata", inputs.metadata); + if(sizeMapping.streamK > 0 && sizeMapping.streamKAtomic == 0) + { + // Assert hardware is not null + // For now grouped gemm is not supported and passes nullptr + TENSILE_ASSERT_EXC(hardware != nullptr); + size_t cuCount = 0; + + auto tiles = problem.getNumTiles(sizeMapping); + size_t skGrid = getSKGrid(*hardware, tiles); + // StreamK workspace + flags + args.template append("ws", inputs.ws); + void* ws = inputs.ws; + size_t flagsOffset = partialTileSize(skGrid); + void* flags = (void*)(static_cast(ws) + flagsOffset); + args.template append("Flags", flags); + } + size_t startStrideCD = problemType.useInitialStridesCD ? 0 : 1; size_t startStrideAB = problemType.useInitialStridesAB ? 0 : 1; - if(gsu > 1 && sizeMapping.globalAccumulation) + if(gsu > 1 && sizeMapping.globalAccumulation && sizeMapping.streamK == 0) { size_t wsStride = startStrideCD ? d.sizes()[0] : 1; for(size_t i = startStrideCD; i < d.dimensions(); i++) @@ -801,7 +819,8 @@ namespace Tensile template KernelInvocation ContractionSolution::generateSingleCall(ContractionSolution::Problem const& problem, - ContractionInputs const& inputs) const + ContractionInputs const& inputs, + Hardware const& hardware) const { TENSILE_ASSERT_EXC(sizeMapping.workGroupMapping >= 0); @@ -847,10 +866,33 @@ namespace Tensile rv.numWorkGroups.x = CeilDivide(rv.numWorkGroups.x, sizeMapping.macroTile.x); rv.numWorkGroups.y = CeilDivide(rv.numWorkGroups.y, sizeMapping.macroTile.y); + uint32_t problemNumGroupTiles0 = rv.numWorkGroups.x; + uint32_t problemNumGroupTiles1 = rv.numWorkGroups.y; + // used only when persistent kernel along batch + uint32_t problemNumGroupTiles2 = rv.numWorkGroups.z; + uint32_t gsu = problem.getParams().gsu() > 0 ? problem.getParams().gsu() : sizeMapping.globalSplitU; rv.numWorkGroups.y *= gsu; + size_t cuCount = 0; + size_t skGrid = 0; + auto tiles = problem.getNumTiles(sizeMapping); + if(sizeMapping.streamK != 0 || sizeMapping.persistentKernel != 0) + { + AMDGPU const* pAMDGPU = dynamic_cast(&hardware); + assert(pAMDGPU != nullptr && pAMDGPU->computeUnitCount != 0); + cuCount = pAMDGPU->computeUnitCount; + if(sizeMapping.streamK != 0) + { + skGrid = getSKGrid(hardware, tiles); + // TODO enable grid size change once code gen changes are implemented + // rv.numWorkGroups.x = skGrid; + // rv.numWorkGroups.y = 1; + // rv.numWorkGroups.z = 1; + } + } + rv.numWorkItems.x = rv.workGroupSize.x * rv.numWorkGroups.x; rv.numWorkItems.y = rv.workGroupSize.y * rv.numWorkGroups.y; rv.numWorkItems.z = rv.workGroupSize.z * rv.numWorkGroups.z; @@ -859,7 +901,7 @@ namespace Tensile if(internalArgsSupport.useUniversalArgs) kernelArgs(1, 0, rv.args, problem.getParams()); - singleCallArgs(problem, inputs, 0, rv.args); + singleCallArgs(problem, inputs, 0, &hardware, rv.args); if(sizeMapping.globalAccumulation == 3) { @@ -868,6 +910,68 @@ namespace Tensile rv.args.append("GSUSync", 0); } + if(sizeMapping.persistentKernel != 0 || sizeMapping.streamK != 0) + { + uint32_t magicShift; + rv.args.append("magicNumberProblemNumGroupTiles0", + magicNumber(2, problemNumGroupTiles0, &magicShift)); + rv.args.append("magicShiftProblemNumGroupTiles0", magicShift); + } + + if(sizeMapping.streamK != 0) + { + auto itersPerTile = problem.getItersPerTile(sizeMapping); + auto totalIters = tiles * itersPerTile; + uint32_t magicNumberItersPerTile; + uint32_t magicShiftItersPerTile; + magicNumberItersPerTile = magicNumber(2, itersPerTile, &magicShiftItersPerTile); + + rv.args.append("itersPerTile", itersPerTile); + rv.args.append("magicNumberItersPerTile", magicNumberItersPerTile); + rv.args.append("magicShiftItersPerTile", magicShiftItersPerTile); + + uint32_t numGroupTiles0x1 = problemNumGroupTiles0 * problemNumGroupTiles1; + uint32_t magicNumProblemNumGroupTiles0By1; + uint32_t magicShiftProblemNumGroupTiles0By1; + magicNumProblemNumGroupTiles0By1 + = magicNumber(2, numGroupTiles0x1, &magicShiftProblemNumGroupTiles0By1); + rv.args.append("magicNumProblemNumGroupTiles0By1", + magicNumProblemNumGroupTiles0By1); + rv.args.append("magicShiftProblemNumGroupTiles0By1", + magicShiftProblemNumGroupTiles0By1); + + rv.args.append("totalIters", totalIters); + if(sizeMapping.streamK == 1) // Basic SK + { + uint32_t itersPerWave = CeilDivide(totalIters, rv.numWorkGroups.x); + rv.args.append("SKItersPerWG", itersPerWave); + } + else if(sizeMapping.streamK >= 2) // Two-tile SK + { + bool bigEnough = tiles > skGrid; + // skTiles is number of Stream-K tiles to complete + // Two-tile algorithm causes each WG to run an even number of Stream-K iterations, + // followed by an even number of data-parllel tiles. + // If total tiles is evenly divisble by grid size, + // then no Stream-K tiles are needed, all data-parallel + uint32_t skTiles = skGrid; + if(tiles % skGrid != 0) + { + // Number of data-parallel tiles on each workgroup would be: + // dpTilesPerWG = bigEnough ? (tiles - skTiles) / skGrid : 0; + skTiles = bigEnough ? skGrid + tiles % skGrid : tiles; + } + + uint32_t skItersPerWG = skTiles * itersPerTile / skGrid; + uint32_t skExtraIters = skTiles * itersPerTile % (skGrid); + + rv.args.append("SKItersPerWG", skItersPerWG); + rv.args.append("skGrid", skGrid); + rv.args.append("skTiles", skTiles); + rv.args.append("skExtraIters", skExtraIters); + } + } + if(problemType.stochasticRounding) { // generate seed from random generator @@ -988,7 +1092,7 @@ namespace Tensile { auto problem = problems[idx]; singleCallArgs( - problem, inputs.grouped[idx], workspaceOffsetInByte, h_args); + problem, inputs.grouped[idx], workspaceOffsetInByte, nullptr, h_args); uint32_t gsu = problem.getParams().gsu() > 0 ? problem.getParams().gsu() : sizeMapping.globalSplitU; @@ -2152,9 +2256,9 @@ namespace Tensile } if(debug) - rv.push_back(generateSingleCall(problem, inputs)); + rv.push_back(generateSingleCall(problem, inputs, hardware)); else - rv.push_back(generateSingleCall(problem, inputs)); + rv.push_back(generateSingleCall(problem, inputs, hardware)); if((sizeMapping.globalAccumulation != 3) && gsu > 1 && sizeMapping.globalAccumulation) From 1e199151d2c8deacd5f7caf9a6e485c09c766807 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Thu, 6 Jun 2024 15:49:06 +0000 Subject: [PATCH 04/36] Increase default max workspace for tensile tuning client --- tensilelite/Tensile/Common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index 807829282e..6670c4390d 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -256,7 +256,7 @@ globalParameters["PerfModelReadEfficiency"] = 0.85 # limitation for training -globalParameters["MaxWorkspaceSize"] = 32 * 1024 * 1024 # max workspace for training (32M) +globalParameters["MaxWorkspaceSize"] = 128 * 1024 * 1024 # max workspace for training (128M) globalParameters["MinKForGSU"] = 32 # min K size to use GlobalSplitU algorithm (only for HPA now) # control if a solution is run for a given problem From 223aedc6d7a93710f24522d39da9c9bb7f6d20b6 Mon Sep 17 00:00:00 2001 From: Bryant Nelson Date: Thu, 6 Jun 2024 13:49:56 -0500 Subject: [PATCH 05/36] Increase default workspace for bench client (#2) --- clients/benchmarks/client_groupedgemm_fixed_mk.cpp | 2 +- clients/benchmarks/client_sequence.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/benchmarks/client_groupedgemm_fixed_mk.cpp b/clients/benchmarks/client_groupedgemm_fixed_mk.cpp index 23ead327bc..fd4d746a8f 100644 --- a/clients/benchmarks/client_groupedgemm_fixed_mk.cpp +++ b/clients/benchmarks/client_groupedgemm_fixed_mk.cpp @@ -770,7 +770,7 @@ int test_hipblaslt(hipDataType in_datatype, } // Set User Preference attributes - uint64_t max_workspace_size = 32 * 1024 * 1024; + uint64_t max_workspace_size = 128 * 1024 * 1024; void* d_workspace; CHECK_HIP_ERROR(hipMalloc(&d_workspace, max_workspace_size)); diff --git a/clients/benchmarks/client_sequence.cpp b/clients/benchmarks/client_sequence.cpp index c9f526cd16..30c5cb2f53 100644 --- a/clients/benchmarks/client_sequence.cpp +++ b/clients/benchmarks/client_sequence.cpp @@ -247,7 +247,7 @@ class LayerConfigIOGeneralSettings uint32_t rotating = 0; // Size in MB uint32_t cold_iters = 1000; uint32_t iters = 10; - int64_t max_workspace_size = 32 * 1024 * 1024; + int64_t max_workspace_size = 128 * 1024 * 1024; bool graph_mode = false; }; From 7809986e0824d473430c5d7f77559621e62dc40a Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Fri, 21 Jun 2024 15:09:04 +0000 Subject: [PATCH 06/36] Update gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 809e9b3af6..ad9dcb67fc 100644 --- a/.gitignore +++ b/.gitignore @@ -36,12 +36,13 @@ tags .vscode # build-in-source directory and documentation artifacts -build/ +build*/ _build/ _images/ _static/ _templates/ _toc.yml +Tensile.egg-info # matrices *.csr From cbf4ef39e7d00616ddcd99394ead081a19bdee44 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Fri, 21 Jun 2024 21:11:32 +0000 Subject: [PATCH 07/36] Option to disable GSU in kernels. Modularize GSU code so it is more contained, reduces branch points in KernelWriter code. Fix a temp register allocation bug. Fix alignment of some registers rather than rely on a specific order. --- tensilelite/Tensile/Common.py | 3 +- tensilelite/Tensile/Components/GSU.py | 410 ++++++++++++++++++ tensilelite/Tensile/Components/__init__.py | 5 +- tensilelite/Tensile/KernelWriter.py | 96 +--- tensilelite/Tensile/KernelWriterAssembly.py | 239 +++------- tensilelite/Tensile/SolutionStructs.py | 4 +- .../Source/lib/source/ContractionSolution.cpp | 14 +- .../Tensile/Tests/common/gsu/f32_gsu.yaml | 58 +++ 8 files changed, 558 insertions(+), 271 deletions(-) create mode 100644 tensilelite/Tensile/Components/GSU.py create mode 100644 tensilelite/Tensile/Tests/common/gsu/f32_gsu.yaml diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index 6670c4390d..5347442244 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -509,7 +509,8 @@ def getArchitectureName(gfxName): # Split the unroll summation into multiple sections and combine the sections # GSU applies only to the unroll summation dimension - "GlobalSplitU": list(range(1, 1024+1)), + # Set to 0 to disable GSU, kernel code will be generated without GSU support + "GlobalSplitU": list(range(0, 1024+1)), # choose how to do GlobalSplitU # 1: use atomic operation to accumulate on one buffer diff --git a/tensilelite/Tensile/Components/GSU.py b/tensilelite/Tensile/Components/GSU.py new file mode 100644 index 0000000000..f2ee2ee311 --- /dev/null +++ b/tensilelite/Tensile/Components/GSU.py @@ -0,0 +1,410 @@ +################################################################################ +# +# Copyright (C) 2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import Module, Label, RegisterPoolResource, SAddU32, SAddCU32, SCmpEQU32, SCBranchSCC1, \ + scalarUInt32DivideAndRemainder, SMovB32, SMulI32, SBranch, SMovB64, SLShiftRightB32, sgpr, log2, \ + SCmpLtU32, SCMovB32, SSubU32, SLShiftLeftB64, SCBranchSCC0, fastdeepcopy, Instruction, SCmpLgU32 +from ..Component import Component +import abc + +class GSU(Component): + """ + GSU block. + """ + @abc.abstractmethod + def graWorkGroup(self, writer, kernel): + pass + + @abc.abstractmethod + def computeLoadSrd(self, writer, kernel, tP, stmp, tileStart): + pass + + @abc.abstractmethod + def graIncrements(self, writer, kernel, loopIdx, tP): + pass + + def graIncrementsCommon(self, writer, loopIdx, tc, stride, m): + module = Module("GSU Common graIncrements") + + # multiply by stride, optimizing if unit stride + if writer.isConstUnitStride(stride): + module.add(SMovB32(dst=sgpr("GlobalReadIncs%s+%u"%(tc, loopIdx)), src=m, \ + comment="incr%s (unrollIdx)"%(tc) )) + else: + module.add(SMulI32(dst=sgpr("GlobalReadIncs%s+%u"%(tc, loopIdx)), \ + src0=m, src1=stride, \ + comment="incr%s unrollIdx)"%(tc) )) + + return module + + @abc.abstractmethod + def calculateLoopNumIter(self, writer, kernel, loopCounterName, tmpSgprInfo): + pass + + @abc.abstractmethod + def computeStoreSrdStart(self, writer, kernel): + pass + + @abc.abstractmethod + def noLoadLoop(self, writer, kernel, tensorParametersA, tensorParametersB, pack): + pass + + @abc.abstractmethod + def tailLoopNumIter(self, writer, kernel, loopCounter): + pass + + @abc.abstractmethod + def setupNewTile(self, writer, kernel, tensorParametersA, tensorParametersB, tPM): + pass + + def graIncrementsAB(self, writer, kernel, tensorParametersA, tensorParametersB, tPM): + module = Module("GSU Common graIncrementsAB") + + for i in reversed(range(kernel["ProblemType"]["NumIndicesSummation"])): + module.add(writer.graIncrements(kernel, i, tensorParametersA)) + if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: + module.addComment1("global read addresses: increments metadata") + for i in reversed(range(kernel["ProblemType"]["NumIndicesSummation"])): + module.add(writer.graIncrements(kernel, i, tPM)) + module.addComment1("global read addresses: increments b") + for i in reversed(range(kernel["ProblemType"]["NumIndicesSummation"])): + module.add(writer.graIncrements(kernel, i, tensorParametersB)) + + return module + +class GSUOff(GSU): + kernel = {"GlobalSplitU": 0} + + def __call__(self): + assert(0) + + def graWorkGroup(self, writer, kernel): + module = Module("GSU Off graWorkGroup") + return module + + def computeLoadSrd(self, writer, kernel, tP, stmp, tileStart): + module = Module("GSU Off computeLoadSrd") + return module + + def graIncrements(self, writer, kernel, loopIdx, tP): + module = Module("GSU Off graIncrements") + + tc = tP["tensorChar"] + dimIdx = kernel["ProblemType"]["IndicesSummation"][loopIdx] # dimension index + stride = writer.strideRef(tc, dimIdx) + isMirrorIdx = dimIdx in kernel["ProblemType"]["MirrorDims%s"%tc] + + m = "DepthU*Bpe%s"%(tc) + if isMirrorIdx: + m = "-%s"%(m) + + module.add(self.graIncrementsCommon(writer, loopIdx, tc, stride, m)) + + return module + + def calculateLoopNumIter(self, writer, kernel, loopCounterName, tmpSgprInfo): + module = Module("GSU Off calculateLoopNumIter") + return module + + def computeStoreSrdStart(self, writer, kernel): + module = Module("GSU Off computeStoreSrdStart") + return module + + def noLoadLoop(self, writer, kernel, tensorParametersA, tensorParametersB, pack): + module = Module("GSU Off noLoadLoop") + return module + + def tailLoopNumIter(self, writer, kernel, loopCounter): + module = Module("GSU Off tailLoopNumIter") + return module + + def setupNewTile(self, writer, kernel, tensorParametersA, tensorParametersB, tPM): + module = Module("GSU Off setupNewTile") + + module.add(self.graIncrementsAB(writer, kernel, tensorParametersA, tensorParametersB, tPM)) + + return module + +class GSUOn(GSU): + + @classmethod + def matches(cls, writer, debug=False): + return writer.states.kernel["GlobalSplitU"] > 0 + + def __call__(self): + assert(0) + + def graWorkGroup(self, writer, kernel): + module = Module("GSU On graWorkGroup") + + gsuLabel = Label(label=writer.labels.getNameInc("GSU"), comment="") + gsuLabelEnd = Label(label=writer.labels.getNameInc("GSU_End"), comment="") + module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) + module.add(SCBranchSCC1(labelName=gsuLabel.getLabelName(), comment="branch if GSU == 1")) + module.addComment("GSU-not-WGMapRR :nwg1 = (size%s + MT%s - 1) / MT%s;" \ + % (writer.states.tileChar1, writer.states.tileChar1, writer.states.tileChar1)) + + # gsuSumIdx = wg1 % GSU + # wg1 = wg1 / GSU + divisor = "WorkGroup1" + tmpVgpr = writer.vgprPool.checkOut(2, "tmp") + tmpVgprRes = RegisterPoolResource(idx=tmpVgpr, size=2) + module.add(scalarUInt32DivideAndRemainder("WorkGroup1", divisor, "GSU", "GSUSumIdx", tmpVgprRes, wavewidth=kernel["WavefrontSize"])) + writer.vgprPool.checkIn(tmpVgpr) + module.add(SMovB32(dst=sgpr("GSULog2BpeC"), src=log2(int(writer.states.bpr * kernel["ProblemType"]["DestDataType"].numRegisters())))) + module.add(SMovB32(dst=sgpr("GSULog2BpeD"), src=log2(writer.states.bpeCinternal))) + + module.add(SBranch(gsuLabelEnd.getLabelName())) + module.add(gsuLabel) + module.add(SMovB64(dst=sgpr("GSUSumIdx", 2), src=0, comment="Set GSUSumIdx to 0")) + module.add(SMovB32(dst=sgpr("GSULog2BpeC"), src=log2(writer.states.bpeCexternalGSU1))) + module.add(SMovB32(dst=sgpr("GSULog2BpeD"), src=log2(writer.states.bpeCexternalGSU1))) + module.add(gsuLabelEnd) + + return module + + def computeLoadSrd(self, writer, kernel, tP, stmp, tileStart): + module = Module("GSU On computeLoadSrd") + + tc = tP["tensorChar"] + depthU = kernel["DepthU"] + gsuOffset_str = "gsuOffset = DepthU*bpeGR*GSUSumIdx" + divider = 1 + if kernel["ProblemType"]["Sparse"]: + if (kernel["ProblemType"]["Sparse"] == 2 and tP["isB"]) or \ + (kernel["ProblemType"]["Sparse"] == 1 and tP["isA"]) : + divider = 2 + elif tP["isM"]: + divider = 8 + if divider != 1: + depthU = depthU // divider + gsuOffset_str = "gsuOffset = DepthU/%s*bpeGR*GSUSumIdx"%(divider) + module.addModuleAsFlatItems(writer.s_mul_u64_u32(sgpr(stmp+0), sgpr(stmp+1), depthU, sgpr("GSUSumIdx"), gsuOffset_str)) + + unrollSummation = [ i for i in tP["ia"] if i in kernel["ProblemType"]["IndicesSummation"] ] + stride = writer.strideRef(tc,unrollSummation[-1]) + if tP["tlu"] and not writer.isConstUnitStride(stride): + # non-transpose case, unroll is in perp dim and should be scaled by unroll Stride + module.addModuleAsFlatItems(writer.s_mul_u64_u32(sgpr(stmp), sgpr(stmp+1), sgpr(stmp+0), \ + stride, "tlu=1, scaled unroll-offset by stride")) + + module.add(SAddU32(dst=sgpr(tileStart+0), src0=sgpr(tileStart+0), src1=sgpr(stmp+0), comment="accum GsuOffset term to tilestart")) + module.add(SAddCU32(dst=sgpr(tileStart+1), src0=sgpr(tileStart+1), src1=sgpr(stmp+1), comment="accum GsuOffset term to tilestart")) + + return module + + def graIncrements(self, writer, kernel, loopIdx, tP): + module = Module("GSU On graIncrements") + + tc = tP["tensorChar"] + dimIdx = kernel["ProblemType"]["IndicesSummation"][loopIdx] # dimension index + stride = writer.strideRef(tc, dimIdx) + isMirrorIdx = dimIdx in kernel["ProblemType"]["MirrorDims%s"%tc] + + with writer.allocTmpSgpr(1) as tmpSgprInfo: + gsuSgpr = tmpSgprInfo.idx + + tcGR = tc if tc == "Metadata" else (tc + "GR") + module.add(SMulI32(dst=sgpr(gsuSgpr), src0=sgpr("GSU"), src1="DepthU*Bpe%s"%(tcGR))) + + if kernel["ProblemType"]["Sparse"]: + if tP["is_sparse"]: + module.add(SLShiftRightB32(dst=sgpr(gsuSgpr), shiftHex=hex(log2(2)), src=sgpr(gsuSgpr))) + elif tP["isM"]: + module.add(SLShiftRightB32(dst=sgpr(gsuSgpr), shiftHex=hex(log2(8)), src=sgpr(gsuSgpr))) + + m = sgpr(gsuSgpr) + + if isMirrorIdx: + m.setMinus(True) + + module.add(self.graIncrementsCommon(writer, loopIdx, tc, stride, m)) + + return module + + def calculateLoopNumIter(self, writer, kernel, loopCounterName, tmpSgprInfo): + module = Module("GSU On calculateLoopNumIter") + + # if GSU numIter++ if gsuSumIdx < remainder + gsuLabel = Label(label=writer.labels.getNameInc("GSU"), comment="") + module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) + module.add(SCBranchSCC1(labelName=gsuLabel.getLabelName(), comment="branch if GSU == 1")) + module.add(self.calculateLoopNumIterGsu(writer, kernel, loopCounterName, tmpSgprInfo)) + module.add(gsuLabel) + + return module + + ############################################################################## + # Emit code to compute loop iterations for GSU. + # See same function in KernelWriterSource.py for background explanation + # This function is used to compute number of loop iters and also + # for computing the global read increment for GSU case. + # For multiple summation, the number of loop iterations needs to be reset + # for each iteration so replicate the code in addr inc and at open of unroll loop + + # tmpSgpr is allocation of at least 3 tmpSgpr + + # Output: SGPR(destName) contains the number of unroll iterations for + # this workgroup. + ############################################################################## + def calculateLoopNumIterGsu(self, writer, kernel, destName, tmpSgprRes: RegisterPoolResource): + module = Module("calculateLoopNumIterGsu") + + loopCounter = sgpr(destName) + quotient = destName + remainder = "GSUSumIdx+1" # numIterPerWgRemainder + dividend = destName + + tmpVgpr = writer.vgprPool.checkOut(2,"tmp") + tmpVgprRes = RegisterPoolResource(idx=tmpVgpr, size=2) + module.add(scalarUInt32DivideAndRemainder(quotient, dividend, "GSU", remainder, tmpVgprRes, wavewidth=kernel["WavefrontSize"])) + writer.vgprPool.checkIn(tmpVgpr) + + # if gsuSumIdx < numIterPerWgRemainder + module.add(SAddU32(dst=sgpr(tmpSgprRes.idx), src0=1, \ + src1=loopCounter, comment="tmp<-numIterMyWg+" )) + module.add(SCmpLtU32(src0=sgpr("GSUSumIdx"), src1=sgpr("GSUSumIdx+1"), \ + comment="gsuSumIdx < numIterPerWgRemainder" )) + module.add(SCMovB32(dst=loopCounter, src=sgpr(tmpSgprRes.idx), comment="numIterMyWg++ if needed")) + + return module + + def computeStoreSrdStart(self, writer, kernel): + module = Module("GSU On computeStoreSrdStart") + + indices = list(range(0, kernel["ProblemType"]["NumIndicesC"])) + numDim = len(indices) + + if kernel["GlobalSplitUAlgorithm"] == 'MultipleBuffer' or kernel["_GlobalAccumulation"] == 'MultipleBufferSingleKernel': + gsuLabel = Label(label=writer.labels.getNameInc("GSU"), comment="") + module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) + module.add(SCBranchSCC1(labelName=gsuLabel.getLabelName(), comment="branch if GSU == 1")) + # GSU algorithm 2: adjust output buffer address to per GSU buffer + with writer.allocTmpSgpr(4, alignment=1) as tmpSgprInfo: + if tmpSgprInfo.idx % 2 == 0: + tmpSgprX2 = tmpSgprInfo.idx+0 + tmpSgpr0 = tmpSgprInfo.idx+0 + tmpSgpr1 = tmpSgprInfo.idx+1 + tmpSgpr2 = tmpSgprInfo.idx+2 + tmpSgpr3 = tmpSgprInfo.idx+3 + else: + tmpSgprX2 = tmpSgprInfo.idx+1 + tmpSgpr0 = tmpSgprInfo.idx+1 + tmpSgpr1 = tmpSgprInfo.idx+2 + tmpSgpr2 = tmpSgprInfo.idx+0 + tmpSgpr3 = tmpSgprInfo.idx+3 + module.addComment("GSU Output Buffer offset: Free0 + (Free1-1)*StrideC1J + (Free2-1)*StrideCK * GSUIdx * bpe%s") + module.addModuleAsFlatItems(writer.s_mul_u64_u32(sgpr(tmpSgpr0), sgpr(tmpSgpr1), sgpr("SizesFree+0"), sgpr("GSUSumIdx"), "Free0")) + for i in range(1, numDim): + module.add(SSubU32(dst=sgpr(tmpSgpr2), src0=sgpr("SizesFree+%u"%i), src1=1, comment="Free%u" % i)) + module.add(SMulI32(dst=sgpr(tmpSgpr2), src0=sgpr(tmpSgpr2), src1=sgpr("GSUSumIdx"), comment="Free%u" % i)) + module.addModuleAsFlatItems(writer.s_mul_u64_u32(sgpr(tmpSgpr2), sgpr(tmpSgpr3), sgpr(tmpSgpr2), sgpr("StrideC%s"%writer.states.indexChars[i]), "Free%u" % i)) + module.add(SAddU32(dst=sgpr(tmpSgpr0), src0=sgpr(tmpSgpr0), src1=sgpr(tmpSgpr2), comment="Free%u" % i)) + module.add(SAddCU32(dst=sgpr(tmpSgpr1), src0=sgpr(tmpSgpr1), src1=sgpr(tmpSgpr3), comment="Free%u" % i)) + module.add(SLShiftLeftB64(dst=sgpr(tmpSgprX2,2), src=sgpr(tmpSgprX2,2), shiftHex=log2(writer.states.bpeCinternal), comment="scale by bpe")) + module.add(SAddU32(dst=sgpr("SrdD+0"), src0=sgpr("SrdD+0"), src1=sgpr(tmpSgprX2), comment="add lo GSU offset to SRD")) + module.add(SAddCU32(dst=sgpr("SrdD+1"), src0=sgpr("SrdD+1"), src1=sgpr(tmpSgpr1), comment="add hi GSU offset to SRD")) + module.add(gsuLabel) + + return module + + def noLoadLoop(self, writer, kernel, tensorParametersA, tensorParametersB, pack): + module = Module("GSU On noLoadLoop") + + gsuLabel = Label(label=writer.labels.getNameInc("GSU"), comment="") + module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) + noLoadLoopModules = None + acclen = 0 + gsuBackup = kernel["GlobalSplitU"] + gsuAccumBackup = kernel["_GlobalAccumulation"] + bpeCexternalBackup = writer.states.bpeCexternal + kernel["GlobalSplitU"] = 1 + kernel["_GlobalAccumulation"] = None + writer.states.bpeCexternal = writer.states.bpeCexternalGSU1 + if kernel["KernelLanguage"] == "Assembly" and kernel["OptNoLoadLoop"] and \ + kernel["BufferLoad"] and kernel["BufferStore"] and writer.states.doShadowInit and \ + kernel["LocalSplitU"]==1 and \ + writer.states.actualSummationLoops==1: + + # two different noLoadLoops: + # 1. OptNLL & PAP global-read interleaved (only for PAP=ON) + # (2. OptNLL : No PAP global-read (For PAP=OFF, or PAP=ON but the last tile)) + # -> this is unified with 1. global-read is invalidated at the last tile. + # 3. OrdinaryNLL (Not Opt.) + writer.saveLocalPointers(kernel, tensorParametersA, tensorParametersB) + # deepCopy packCode for OptNLL noLoadLoop + deepCopyPack = fastdeepcopy(pack) + noLoadLoopModules = writer.noLoadLoop(kernel, tensorParametersA, tensorParametersB, isOptNLL=True, isNGLL=False, pack=deepCopyPack) + acclen = noLoadLoopModules.countType(Instruction) + writer.restoreLocalPointers(kernel, tensorParametersA, tensorParametersB) + kernel["GlobalSplitU"] = gsuBackup + kernel["_GlobalAccumulation"] = gsuAccumBackup + writer.states.bpeCexternal = bpeCexternalBackup + + if acclen > 16384: + with writer.allocTmpSgpr(3) as tmpSgprInfo: + module.add(writer.longBranchScc0(gsuLabel, posNeg=1, tmpSgprInfo=tmpSgprInfo, comment="branch if GSU != 1")) + else: + module.add(SCBranchSCC0(labelName=gsuLabel.getLabelName(), comment="branch if GSU != 1")) + + if noLoadLoopModules != None: + module.add(noLoadLoopModules) + module.add(gsuLabel) + + return module + + def tailLoopNumIter(self, writer, kernel, loopCounter): + module = Module("GSU On tailLoopNumIter") + + # if GSU numIter=0 if gsuSumIdx != remainder + module.add(SCmpLgU32(src0=sgpr("GSUSumIdx"), src1=sgpr("GSUSumIdx+1"), \ + comment="gsuSumIdx == numIterPerWgRemainder")) + module.add(SCMovB32(dst=loopCounter, src=hex(0), comment="numIter=0 if gsuSimIdx!=remainder")) + + return module + + def setupNewTile(self, writer, kernel, tensorParametersA, tensorParametersB, tPM): + module = Module("GSU On setupNewTile") + + addBranch = False + for i in reversed(range(kernel["ProblemType"]["NumIndicesSummation"])): + if i != writer.states.unrollIdx: + addBranch = True + break + if addBranch: + gsuBackup = kernel["GlobalSplitU"] + gsuLabel = Label(label=writer.labels.getNameInc("GSU"), comment="") + gsuLabelEnd = Label(label=writer.labels.getNameInc("GSU_End"), comment="") + module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) + module.add(SCBranchSCC1(labelName=gsuLabel.getLabelName(), comment="branch if GSU == 1")) + module.addComment1("global read addresses: increments a") + kernel["GlobalSplitU"] = 2 + module.add(self.graIncrementsAB(writer, kernel, tensorParametersA, tensorParametersB, tPM)) + if addBranch: + module.add(SBranch(gsuLabelEnd.getLabelName())) + module.add(gsuLabel) + kernel["GlobalSplitU"] = 1 + module.add(self.graIncrementsAB(writer, kernel, tensorParametersA, tensorParametersB, tPM)) + kernel["GlobalSplitU"] = gsuBackup + module.add(gsuLabelEnd) + + return module \ No newline at end of file diff --git a/tensilelite/Tensile/Components/__init__.py b/tensilelite/Tensile/Components/__init__.py index bff3772227..449b92c563 100644 --- a/tensilelite/Tensile/Components/__init__.py +++ b/tensilelite/Tensile/Components/__init__.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -45,5 +45,6 @@ def use(): pass "PackData", "SIA", "GlobalWriteBatch", - "SumUnroll" + "SumUnroll", + "GSU" ] diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 345471a7f2..1f42b6febf 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -1465,40 +1465,9 @@ def setupNewTile(self, kernel, tensorParametersA, tensorParametersB, isOptNLL=Fa module.addComment1("global read addresses: addresses b") module.add(self.graAddresses(kernel, tensorParametersB)) - # increments - def graIncrementsAB(): - for i in reversed(range(kernel["ProblemType"]["NumIndicesSummation"])): - module.add(self.graIncrements(kernel, i, tensorParametersA)) - if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: - module.addComment1("global read addresses: increments metadata") - for i in reversed(range(kernel["ProblemType"]["NumIndicesSummation"])): - module.add(self.graIncrements(kernel, i, tPM)) - module.addComment1("global read addresses: increments b") - for i in reversed(range(kernel["ProblemType"]["NumIndicesSummation"])): - module.add(self.graIncrements(kernel, i, tensorParametersB)) - - addBranch = False - for i in reversed(range(kernel["ProblemType"]["NumIndicesSummation"])): - if i != self.states.unrollIdx: - addBranch = True - break - if addBranch: - gsuBackup = kernel["GlobalSplitU"] - gsuLabel = Label(label=self.labels.getNameInc("GSU"), comment="") - gsuLabelEnd = Label(label=self.labels.getNameInc("GSU_End"), comment="") - module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) - module.add(SCBranchSCC1(labelName=gsuLabel.getLabelName(), comment="branch if GSU == 1")) - module.addComment1("global read addresses: increments a") - kernel["GlobalSplitU"] = 2 - graIncrementsAB() - if addBranch: - module.add(SBranch(gsuLabelEnd.getLabelName())) - module.add(gsuLabel) - kernel["GlobalSplitU"] = 1 - graIncrementsAB() - kernel["GlobalSplitU"] = gsuBackup - module.add(gsuLabelEnd) - + # Add increment code + gsuComponent = Component.GSU.find(self) + module.add(gsuComponent.setupNewTile(self, kernel, tensorParametersA, tensorParametersB, tPM)) self.dontAppendCode = self.dontAppendCode or forceNoTileCode @@ -2232,45 +2201,8 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): # execute the NLL inside each unroll iteration not just once at the end. if kernel["PrefetchGlobalRead"]: if not kernel["SuppressNoLoadLoop"]: - gsuLabel = Label(label=self.labels.getNameInc("GSU"), comment="") - module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) - noLoadLoopModules = None - acclen = 0 - gsuBackup = kernel["GlobalSplitU"] - gsuAccumBackup = kernel["_GlobalAccumulation"] - bpeCexternalBackup = self.states.bpeCexternal - kernel["GlobalSplitU"] = 1 - kernel["_GlobalAccumulation"] = None - self.states.bpeCexternal = self.states.bpeCexternalGSU1 - if kernel["KernelLanguage"] == "Assembly" and kernel["OptNoLoadLoop"] and \ - kernel["BufferLoad"] and kernel["BufferStore"] and self.states.doShadowInit and \ - kernel["LocalSplitU"]==1 and \ - self.states.actualSummationLoops==1: - - # two different noLoadLoops: - # 1. OptNLL & PAP global-read interleaved (only for PAP=ON) - # (2. OptNLL : No PAP global-read (For PAP=OFF, or PAP=ON but the last tile)) - # -> this is unified with 1. global-read is invalidated at the last tile. - # 3. OrdinaryNLL (Not Opt.) - self.saveLocalPointers(kernel, tensorParametersA, tensorParametersB) - # deepCopy packCode for OptNLL noLoadLoop - deepCopyPack = fastdeepcopy(pack) - noLoadLoopModules = self.noLoadLoop(kernel, tensorParametersA, tensorParametersB, isOptNLL=True, isNGLL=False, pack=deepCopyPack) - acclen = noLoadLoopModules.countType(Instruction) - self.restoreLocalPointers(kernel, tensorParametersA, tensorParametersB) - kernel["GlobalSplitU"] = gsuBackup - kernel["_GlobalAccumulation"] = gsuAccumBackup - self.states.bpeCexternal = bpeCexternalBackup - - if acclen > 16384: - with self.allocTmpSgpr(3) as tmpSgprInfo: - module.add(self.longBranchScc0(gsuLabel, posNeg=1, tmpSgprInfo=tmpSgprInfo, comment="branch if GSU != 1")) - else: - module.add(SCBranchSCC0(labelName=gsuLabel.getLabelName(), comment="branch if GSU != 1")) - - if noLoadLoopModules != None: - module.add(noLoadLoopModules) - module.add(gsuLabel) + gsuComponent = Component.GSU.find(self) + module.add(gsuComponent.noLoadLoop(self, kernel, tensorParametersA, tensorParametersB, pack)) module.add(self.noLoadLoop(kernel, tensorParametersA, tensorParametersB, isOptNLL=False, isNGLL=False, pack=pack)) if self.states.actualSummationLoops>1: @@ -3552,9 +3484,10 @@ def readWriteVectors(mat, vw, kernel): if self.db["EnableAsserts"]: self.defineSgpr("SaveExecMask", 2, 2) - self.defineSgpr("GSUSumIdx", 2, 2) - self.defineSgpr("GSULog2BpeC", 1) - self.defineSgpr("GSULog2BpeD", 1) + if kernel["GlobalSplitU"] > 0: + self.defineSgpr("GSUSumIdx", 2, 2) + self.defineSgpr("GSULog2BpeC", 1) + self.defineSgpr("GSULog2BpeD", 1) self.defineSgpr("StaggerU", 1) self.defineSgpr("WGM", 1) @@ -3671,12 +3604,12 @@ def readWriteVectors(mat, vw, kernel): self.states.numSgprBeta = numSgprBeta if GSUAMBSK: - self.defineSgpr("GSUSync", 1) - self.states.numSgprAddressGSUSync += 1 - self.defineSgpr("AddressTD", numSgprAddressD) + self.defineSgpr("AddressTD", numSgprAddressD, align=2) self.states.numSgprAddressGSUSync += numSgprAddressD - self.defineSgpr("Synchronizer", 2) + self.defineSgpr("Synchronizer", 2, align=2) self.states.numSgprAddressGSUSync += 2 + self.defineSgpr("GSUSync", 1) + self.states.numSgprAddressGSUSync += 1 if kernel["StreamK"]: # StreamK args @@ -3697,7 +3630,8 @@ def readWriteVectors(mat, vw, kernel): # self.defineSgpr("dpTilesPerWG", 1, kernarg=True) self.states.numSgprStreamK += 3 - self.defineSgpr("GSU", 1) # Can't move to the front because of the preload arguments + if kernel["GlobalSplitU"] > 0: + self.defineSgpr("GSU", 1) # Can't move to the front because of the preload arguments #------------------------ # Registers defined below this point are not available in the post-loop diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 4bc845357f..72d2d158f4 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -1141,8 +1141,11 @@ def defineAndResources(self, kernel, tPA, tPB, tPM): commonArgs = Module("load arguments") commonArgs.addComment1("Load num of Gemms") commonArgs.add(self.argLoader.loadKernArg(sgprNumsOfGemm, "KernArgAddress", hex(0), dword=1)) - commonArgs.addComment1("Load GSU data") - commonArgs.add(self.argLoader.loadKernArg("GSU", "KernArgAddress", hex(4), dword=1)) + + sgprPackedArgs = self.sgprPool.checkOut(1, preventOverflow=0) + # Load combined internal arguments + commonArgs.addComment1("Load packed kernel args (StaggerU/WGM/GSU)") + commonArgs.add(self.argLoader.loadKernArg(sgprPackedArgs, "KernArgAddress", hex(4), dword=1)) loadedCommonArgs = 8 # TODO: Need to make this automatically calculated ######################################## # kernel args parameters @@ -1224,7 +1227,7 @@ def defineAndResources(self, kernel, tPA, tPB, tPM): moduleArgs.add(SMovB32(dst=sgpr("AddressTD"), src=sgpr(preloadSgprStartIdx+7), comment="Load AddressTD data")) moduleArgs.add(extReadEpilogueLabeltmp) - moduleArgs.add(SMovB32(dst=sgpr("GSU"), src=sgpr(preloadSgprStartIdx+1), comment="Preload internal args")) + moduleArgs.add(SMovB32(dst=sgpr(sgprPackedArgs), src=sgpr(preloadSgprStartIdx+1), comment="Preload internal args")) moduleArgs.add(SCmpEQU32(src0=sgpr(sgprArgType), src1=(0), comment="Is kernel args")) preloadLabelHBM = Label("Preload_HBMArgs", comment="") perloadLabelLoadEnd = Label("Preload_LoadArgsEnd", comment="") @@ -1247,15 +1250,18 @@ def defineAndResources(self, kernel, tPA, tPB, tPM): for i in range(kernel["ProblemType"]["NumIndicesC"]): moduleRegInit.add(SMovB32(dst=sgpr("WorkGroup0+%u"%i), src=sgpr(preloadSgprStartIdx+self.states.numSgprPreload+i), \ comment="restore workgroup id")) - moduleRegInit.add(SAndB32(dst=sgpr("WGM"), src0=sgpr("GSU"), src1=hex(0xFF00), comment="Restore WGM")) + moduleRegInit.add(SAndB32(dst=sgpr("WGM"), src0=sgpr(sgprPackedArgs), src1=hex(0xFF00), comment="Restore WGM")) moduleRegInit.add(SLShiftRightB32(dst=sgpr("WGM"), shiftHex=hex(8), src=sgpr("WGM"))) - moduleRegInit.add(SAndB32(dst=sgpr("StaggerU"), src0=sgpr("GSU"), src1=hex(0xFFFF0000), comment="Restore StaggerU related vars")) + moduleRegInit.add(SAndB32(dst=sgpr("StaggerU"), src0=sgpr(sgprPackedArgs), src1=hex(0xFFFF0000), comment="Restore StaggerU related vars")) moduleRegInit.add(SLShiftRightB32(dst=sgpr("StaggerU"), shiftHex=hex(16), src=sgpr("StaggerU"))) - moduleRegInit.add(SAndB32(dst=sgpr("GSU"), src0=sgpr("GSU"), src1=hex(0xFF), comment="Restore GSU")) + if kernel["GlobalSplitU"] > 0: + moduleRegInit.add(SAndB32(dst=sgpr("GSU"), src0=sgpr(sgprPackedArgs), src1=hex(0xFF), comment="Restore GSU")) if kernel["ProblemType"]["SupportUserArgs"]: moduleRegInit.add(SMovB32(dst=sgpr("ArgType"),src=sgpr(sgprArgType))) + self.sgprPool.checkIn(sgprPackedArgs) + if kernel["StorePriorityOpt"]: moduleRegInit.add(SSetPrior(prior=3, comment="optimization store")) @@ -1549,7 +1555,8 @@ def calculateWG(): # accumulate tiles of each gemm module.add(SMulI32(dst=sgpr(tmpSgprNumWG0), src0=sgpr(tmpSgprNumWG0), src1=sgpr(tmpSgprNumWG1))) module.add(SMulI32(dst=sgpr(tmpSgprNumWG0), src0=sgpr(tmpSgprNumWG0), src1=sgpr(tmpSgprB))) - module.add(SMulI32(dst=sgpr(tmpSgprNumWG0), src0=sgpr(tmpSgprNumWG0), src1=sgpr("GSU"))) + if kernel["GlobalSplitU"] > 0: + module.add(SMulI32(dst=sgpr(tmpSgprNumWG0), src0=sgpr(tmpSgprNumWG0), src1=sgpr("GSU"))) module.add(SAddU32(dst=sgpr(tmpSgprAccumTiles), src0=sgpr(tmpSgprAccumTiles), src1=sgpr(tmpSgprNumWG0))) # check wgIndex >= AccumTiles? module.add(SCmpLtU32(src0=sgpr("WorkGroup0"), src1=sgpr(tmpSgprAccumTiles))) @@ -1588,7 +1595,8 @@ def calculateWG(): # accumulate tiles of each gemm module.add(SMulI32(dst=sgpr(tmpSgprNumWG0), src0=sgpr(tmpSgprNumWG0), src1=sgpr(tmpSgprNumWG1))) module.add(SMulI32(dst=sgpr(tmpSgprNumWG0), src0=sgpr(tmpSgprNumWG0), src1=sgpr(tmpSgprB))) - module.add(SMulI32(dst=sgpr(tmpSgprNumWG0), src0=sgpr(tmpSgprNumWG0), src1=sgpr("GSU"))) + if kernel["GlobalSplitU"] > 0: + module.add(SMulI32(dst=sgpr(tmpSgprNumWG0), src0=sgpr(tmpSgprNumWG0), src1=sgpr("GSU"))) module.add(SAddU32(dst=sgpr(tmpSgprAccumTiles), src0=sgpr(tmpSgprAccumTiles), src1=sgpr(tmpSgprNumWG0))) # gemmIndex found @@ -1702,14 +1710,16 @@ def calculateWG(): regStateRes = RegisterPoolResource(idx=tmpSgpr.idx+1, size=1) tmpVgprRes = RegisterPoolResource(tmpVgpr, 2) module.add(SMulI32(dst=sgpr(tmpSgpr.idx), src0=sgpr("NumWorkGroups0"), src1=sgpr("NumWorkGroups1"))) - module.add(SMulI32(dst=sgpr(tmpSgpr.idx), src0=sgpr(tmpSgpr.idx), src1=sgpr("GSU"))) + if kernel["GlobalSplitU"] > 0: + module.add(SMulI32(dst=sgpr(tmpSgpr.idx), src0=sgpr(tmpSgpr.idx), src1=sgpr("GSU"))) module.add(scalarUInt32DivideAndRemainder(qReg=tmpSgpr.idx, dReg="WorkGroup0", divReg=tmpSgpr.idx, rReg=tmpSgpr.idx+1,\ tmpVgprRes=tmpVgprRes, wavewidth=kernel["WavefrontSize"], doRemainder=False)) module.add(SMovB32(dst=sgpr("WorkGroup2"), src=sgpr(tmpSgpr.idx))) module.addComment0("idxWG01 = idxWG012 - wg2 * numWG0 * numWG1") module.add(SMulI32(dst=sgpr(tmpSgpr.idx), src0=sgpr("NumWorkGroups1"), src1=sgpr("NumWorkGroups0"))) module.add(SMulI32(dst=sgpr(tmpSgpr.idx), src0=sgpr(tmpSgpr.idx), src1=sgpr("WorkGroup2"))) - module.add(SMulI32(dst=sgpr(tmpSgpr.idx), src0=sgpr(tmpSgpr.idx), src1=sgpr("GSU"))) + if kernel["GlobalSplitU"] > 0: + module.add(SMulI32(dst=sgpr(tmpSgpr.idx), src0=sgpr(tmpSgpr.idx), src1=sgpr("GSU"))) module.add(SSubU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=sgpr(tmpSgpr.idx))) module.addComment0("wg1 = idxWG01 * smallMagicNumber(1/numWG0)") module.add(scalarUInt32DivideAndRemainder(qReg=tmpSgpr.idx, dReg="WorkGroup0", divReg="NumWorkGroups0", rReg=tmpSgpr.idx+1,\ @@ -1889,29 +1899,8 @@ def graWorkGroup(self, kernel): module = Module("graWorkGroup") module.addComment0("graWorkGroup mapping") - gsuLabel = Label(label=self.labels.getNameInc("GSU"), comment="") - gsuLabelEnd = Label(label=self.labels.getNameInc("GSU_End"), comment="") - module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) - module.add(SCBranchSCC1(labelName=gsuLabel.getLabelName(), comment="branch if GSU == 1")) - module.addComment("GSU-not-WGMapRR :nwg1 = (size%s + MT%s - 1) / MT%s;" \ - % (self.states.tileChar1, self.states.tileChar1, self.states.tileChar1)) - - # gsuSumIdx = wg1 % GSU - # wg1 = wg1 / GSU - divisor = "WorkGroup1" - tmpVgpr = self.vgprPool.checkOut(2, "tmp") - tmpVgprRes = RegisterPoolResource(idx=tmpVgpr, size=2) - module.add(scalarUInt32DivideAndRemainder("WorkGroup1", divisor, "GSU", "GSUSumIdx", tmpVgprRes, wavewidth=kernel["WavefrontSize"])) - self.vgprPool.checkIn(tmpVgpr) - module.add(SMovB32(dst=sgpr("GSULog2BpeC"), src=log2(int(self.states.bpr * kernel["ProblemType"]["DestDataType"].numRegisters())))) - module.add(SMovB32(dst=sgpr("GSULog2BpeD"), src=log2(self.states.bpeCinternal))) - - module.add(SBranch(gsuLabelEnd.getLabelName())) - module.add(gsuLabel) - module.add(SMovB64(dst=sgpr("GSUSumIdx", 2), src=0, comment="Set GSUSumIdx to 0")) - module.add(SMovB32(dst=sgpr("GSULog2BpeC"), src=log2(self.states.bpeCexternalGSU1))) - module.add(SMovB32(dst=sgpr("GSULog2BpeD"), src=log2(self.states.bpeCexternalGSU1))) - module.add(gsuLabelEnd) + gsuComponent = Component.GSU.find(self) + module.add(gsuComponent.graWorkGroup(self, kernel)) ######################################## # Blocked rows or columns @@ -1926,6 +1915,8 @@ def graWorkGroup(self, kernel): wgSerial2 = tmpSgprInfoList[2].idx wgmDivisorMagicNumber = tmpSgprInfoList[0].idx + 1 + tmpVgpr = self.vgprPool.checkOut(2, "div") + tmpVgprRes = RegisterPoolResource(idx=tmpVgpr, size=2) # note this overwrites blockId2+1 module.add(scalarUInt32DivideAndRemainder(qReg=blockId2, dReg="WorkGroup1", divReg="WGM", rReg=wgSerial2, tmpVgprRes=tmpVgprRes, wavewidth=kernel["WavefrontSize"], doRemainder=False, comment="WGM")) module.add(SMulI32(dst=sgpr(wgSerial2), src0=sgpr(blockId2), src1=sgpr("WGM"), comment="quotient * non-magic divisor")) @@ -1944,8 +1935,6 @@ def graWorkGroup(self, kernel): # No longer supported for kernel["WorkGroupMapping"] < 0 # WorkGroup0 = wgSerial2 / wgmDivisor # WorkGroup1 = wgSerial2 - (wgmDivisor * WorkGroup0) - tmpVgpr = self.vgprPool.checkOut(2, "div") - tmpVgprRes = RegisterPoolResource(idx=tmpVgpr, size=2) module.add(scalarUInt32DivideAndRemainder(qReg="WorkGroup0", dReg=wgSerial2, divReg=wgmDivisor, rReg="WorkGroup1", tmpVgprRes=tmpVgprRes, wavewidth=kernel["WavefrontSize"])) self.vgprPool.checkIn(tmpVgpr) module.add(SMulI32(dst=sgpr(blockId2), src0=sgpr(blockId2), src1=sgpr("WGM"), comment="blockId * WGM")) @@ -2682,30 +2671,9 @@ def computeLoadSrd(self, kernel, tP, tc, indices, bpe): if not self.isConstUnitStride(strideF): module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tileStart), sgpr(tileStart+1), sgpr(tileStart+0), \ strideF, "tlu=0, scaled tile-offset by stride")) - - depthU = kernel["DepthU"] - gsuOffset_str = "gsuOffset = DepthU*bpeGR*GSUSumIdx" - divider = 1 - if kernel["ProblemType"]["Sparse"]: - if (kernel["ProblemType"]["Sparse"] == 2 and tP["isB"]) or \ - (kernel["ProblemType"]["Sparse"] == 1 and tP["isA"]) : - divider = 2 - elif tP["isM"]: - divider = 8 - if divider != 1: - depthU = depthU // divider - gsuOffset_str = "gsuOffset = DepthU/%s*bpeGR*GSUSumIdx"%(divider) - module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(stmp+0), sgpr(stmp+1), depthU, sgpr("GSUSumIdx"), gsuOffset_str)) - - unrollSummation = [ i for i in tP["ia"] if i in kernel["ProblemType"]["IndicesSummation"] ] - stride = self.strideRef(tc,unrollSummation[-1]) - if tP["tlu"] and not self.isConstUnitStride(stride): - # non-transpose case, unroll is in perp dim and should be scaled by unroll Stride - module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(stmp), sgpr(stmp+1), sgpr(stmp+0), \ - stride, "tlu=1, scaled unroll-offset by stride")) - - module.add(SAddU32(dst=sgpr(tileStart+0), src0=sgpr(tileStart+0), src1=sgpr(stmp+0), comment="accum GsuOffset term to tilestart")) - module.add(SAddCU32(dst=sgpr(tileStart+1), src0=sgpr(tileStart+1), src1=sgpr(stmp+1), comment="accum GsuOffset term to tilestart")) + + gsuComponent = Component.GSU.find(self) + module.add(gsuComponent.computeLoadSrd(self, kernel, tP, stmp, tileStart)) # Output : tileStart[0:1] have offset in elements from the 2D start of the tile. # if groOffsetInMacroTile=1, 2DStart + tileStart gives the the start of the macro-tile; @@ -2921,31 +2889,8 @@ def graIncrements(self, kernel, loopIdx, tP): dst=vgpr("GlobalReadIncs%s+%u+1"%(tc, 2*loopIdx)), \ src=sgpr(tmpSgpr+1))) else: # not globalReadIncsUseVgpr, ie use SGPR - with self.allocTmpSgpr(1) as tmpSgprInfo: - gsuSgpr = tmpSgprInfo.idx - - tcGR = tc if tc == "Metadata" else (tc + "GR") - module.add(SMulI32(dst=sgpr(gsuSgpr), src0=sgpr("GSU"), src1="DepthU*Bpe%s"%(tcGR))) - - if kernel["ProblemType"]["Sparse"]: - if tP["is_sparse"]: - module.add(SLShiftRightB32(dst=sgpr(gsuSgpr), shiftHex=hex(log2(2)), src=sgpr(gsuSgpr))) - elif tP["isM"]: - module.add(SLShiftRightB32(dst=sgpr(gsuSgpr), shiftHex=hex(log2(8)), src=sgpr(gsuSgpr))) - - m = sgpr(gsuSgpr) - - if isMirrorIdx: - m.setMinus(True) - - # multiply by stride, optimizing if unit stride - if self.isConstUnitStride(stride): - module.add(SMovB32(dst=sgpr("GlobalReadIncs%s+%u"%(tc, loopIdx)), src=m, \ - comment="incr%s (unrollIdx)"%(tc) )) - else: - module.add(SMulI32(dst=sgpr("GlobalReadIncs%s+%u"%(tc, loopIdx)), \ - src0=m, src1=stride, \ - comment="incr%s unrollIdx)"%(tc) )) + gsuComponent = Component.GSU.find(self) + module.add(gsuComponent.graIncrements(self, kernel, loopIdx, tP)) else: # other summation if self.states.globalReadIncsUseVgpr: @@ -2983,7 +2928,8 @@ def graIncrements(self, kernel, loopIdx, tP): divisor, tmpSgprInfo, 0)) if kernel["GlobalSplitU"] > 1: - module.add(self.calculateLoopNumIterGsu(kernel, loopCounterName, tmpSgprInfo)) + gsuComponent = Component.GSU.find(self) + module.add(gsuComponent.calculateLoopNumIterGsu(self, kernel, loopCounterName, tmpSgprInfo)) with self.allocTmpSgpr(1) as tmpSgprInfo: gsuSgpr = tmpSgprInfo.idx @@ -3691,41 +3637,6 @@ def removeStagger(self, kernel, tP): return imod - ############################################################################## - # Emit code to compute loop iterations for GSU. - # See same function in KernelWriterSource.py for background explanation - # This function is used to compute number of loop iters and also - # for computing the global read increment for GSU case. - # For multiple summation, the number of loop iterations needs to be reset - # for each iteration so replicate the code in addr inc and at open of unroll loop - - # tmpSgpr is allocation of at least 3 tmpSgpr - - # Output: SGPR(destName) contains the number of unroll iterations for - # this workgroup. - ############################################################################## - def calculateLoopNumIterGsu(self, kernel, destName, tmpSgprRes: RegisterPoolResource): - module = Module("calculateLoopNumIterGsu") - - loopCounter = sgpr(destName) - quotient = destName - remainder = "GSUSumIdx+1" # numIterPerWgRemainder - dividend = destName - - tmpVgpr = self.vgprPool.checkOut(2,"tmp") - tmpVgprRes = RegisterPoolResource(idx=tmpVgpr, size=2) - module.add(scalarUInt32DivideAndRemainder(quotient, dividend, "GSU", remainder, tmpVgprRes, wavewidth=kernel["WavefrontSize"])) - self.vgprPool.checkIn(tmpVgpr) - - # if gsuSumIdx < numIterPerWgRemainder - module.add(SAddU32(dst=sgpr(tmpSgprRes.idx), src0=1, \ - src1=loopCounter, comment="tmp<-numIterMyWg+" )) - module.add(SCmpLtU32(src0=sgpr("GSUSumIdx"), src1=sgpr("GSUSumIdx+1"), \ - comment="gsuSumIdx < numIterPerWgRemainder" )) - module.add(SCMovB32(dst=loopCounter, src=sgpr(tmpSgprRes.idx), comment="numIterMyWg++ if needed")) - - return module - ############################################################################## # Calculate Loop Num Iter # loopIdx is the index of the loop (used for contractions with multiple summations) @@ -3805,10 +3716,8 @@ def calculateLoopNumIter(self, kernel, tPA, tPB, loopIdx): self.vgprPool.checkIn(wave_id) self.vgprPool.checkIn(tmpVgpr) - # if GSU numIter=0 if gsuSumIdx != remainder - module.add(SCmpLgU32(src0=sgpr("GSUSumIdx"), src1=sgpr("GSUSumIdx+1"), \ - comment="gsuSumIdx == numIterPerWgRemainder")) - module.add(SCMovB32(dst=loopCounter, src=hex(0), comment="numIter=0 if gsuSimIdx!=remainder")) + gsuComponent = Component.GSU.find(self) + module.add(gsuComponent.tailLoopNumIter(self, kernel, loopCounter)) # if tail numIter == 0 skip altogether skipTailLoopLabel = Label.getFormatting("SkipTailLoop%s"%(loopChar) ) @@ -3840,12 +3749,9 @@ def calculateLoopNumIter(self, kernel, tPA, tPB, loopIdx): module.add(scalarStaticDivideAndRemainder(quotient, None, quotient, divisor, tmpSgprInfo, 0)) else: module.add(scalarStaticDivideAndRemainder(quotient, None, dividend, divisor, tmpSgprInfo, 0)) - # if GSU numIter++ if gsuSumIdx < remainder - gsuLabel = Label(label=self.labels.getNameInc("GSU"), comment="") - module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) - module.add(SCBranchSCC1(labelName=gsuLabel.getLabelName(), comment="branch if GSU == 1")) - module.add(self.calculateLoopNumIterGsu(kernel, loopCounterName, tmpSgprInfo)) - module.add(gsuLabel) + + gsuComponent = Component.GSU.find(self) + module.add(gsuComponent.calculateLoopNumIter(self, kernel, loopCounterName, tmpSgprInfo)) module.add(SMovB32(dst=sgpr("OrigLoopCounter"), \ src=loopCounter, \ @@ -4336,7 +4242,7 @@ def fixPreloadOffset(offset, sgpxIdxVec, numStoreSgprToLoad): sgpxIdxVec = self.defineMultiSgprIndex(self.states.numStoreSgprNames, self.states.numStoreSgprNameSizes, align=4) for name in self.states.numStoreSgprNames: module.add(RegSet("s", "sgpr"+name, self.sgprs[name])) - if noSkipLoad: + if noSkipLoad and kernel["GlobalSplitU"] > 0: gsuLabel = Label(label=self.labels.getNameInc("GSU"), comment="") module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) if (kernel["_GlobalAccumulation"] != 'MultipleBufferSingleKernel'): @@ -4439,7 +4345,7 @@ def fixPreloadOffset(offset, sgpxIdxVec, numStoreSgprToLoad): loadModule = module.addModuleAsFlatItems(self.argLoader.loadAllKernArg(startVgprName, "KernArgAddress", numStoreSgprToLoad)) self.states.numStoreSgprInst = loadModule.countType(SMemLoadInstruction) self.argLoader.setOffset(argOffset) # Restore offset - if noSkipLoad: + if noSkipLoad and kernel["GlobalSplitU"] > 0: module.add(gsuLabel) ######################################## @@ -7467,36 +7373,8 @@ def computeStoreSrdStart(self, kernel, srdTcList: list, sgprBpeList = [], useSiz if noMultipleBuffer: return module - if kernel["GlobalSplitUAlgorithm"] == 'MultipleBuffer' or kernel["_GlobalAccumulation"] == 'MultipleBufferSingleKernel': - gsuLabel = Label(label=self.labels.getNameInc("GSU"), comment="") - module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) - module.add(SCBranchSCC1(labelName=gsuLabel.getLabelName(), comment="branch if GSU == 1")) - # GSU algorithm 2: adjust output buffer address to per GSU buffer - with self.allocTmpSgpr(4, alignment=1) as tmpSgprInfo: - if tmpSgprInfo.idx % 2 == 0: - tmpSgprX2 = tmpSgprInfo.idx+0 - tmpSgpr0 = tmpSgprInfo.idx+0 - tmpSgpr1 = tmpSgprInfo.idx+1 - tmpSgpr2 = tmpSgprInfo.idx+2 - tmpSgpr3 = tmpSgprInfo.idx+3 - else: - tmpSgprX2 = tmpSgprInfo.idx+1 - tmpSgpr0 = tmpSgprInfo.idx+1 - tmpSgpr1 = tmpSgprInfo.idx+2 - tmpSgpr2 = tmpSgprInfo.idx+0 - tmpSgpr3 = tmpSgprInfo.idx+3 - module.addComment("GSU Output Buffer offset: Free0 + (Free1-1)*StrideC1J + (Free2-1)*StrideCK * GSUIdx * bpe%s") - module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tmpSgpr0), sgpr(tmpSgpr1), sgpr("SizesFree+0"), sgpr("GSUSumIdx"), "Free0")) - for i in range(1, numDim): - module.add(SSubU32(dst=sgpr(tmpSgpr2), src0=sgpr("SizesFree+%u"%i), src1=1, comment="Free%u" % i)) - module.add(SMulI32(dst=sgpr(tmpSgpr2), src0=sgpr(tmpSgpr2), src1=sgpr("GSUSumIdx"), comment="Free%u" % i)) - module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tmpSgpr2), sgpr(tmpSgpr3), sgpr(tmpSgpr2), sgpr("StrideC%s"%self.states.indexChars[i]), "Free%u" % i)) - module.add(SAddU32(dst=sgpr(tmpSgpr0), src0=sgpr(tmpSgpr0), src1=sgpr(tmpSgpr2), comment="Free%u" % i)) - module.add(SAddCU32(dst=sgpr(tmpSgpr1), src0=sgpr(tmpSgpr1), src1=sgpr(tmpSgpr3), comment="Free%u" % i)) - module.add(SLShiftLeftB64(dst=sgpr(tmpSgprX2,2), src=sgpr(tmpSgprX2,2), shiftHex=log2(self.states.bpeCinternal), comment="scale by bpe")) - module.add(SAddU32(dst=sgpr("SrdD+0"), src0=sgpr("SrdD+0"), src1=sgpr(tmpSgprX2), comment="add lo GSU offset to SRD")) - module.add(SAddCU32(dst=sgpr("SrdD+1"), src0=sgpr("SrdD+1"), src1=sgpr(tmpSgpr1), comment="add hi GSU offset to SRD")) - module.add(gsuLabel) + gsuComponent = Component.GSU.find(self) + module.add(gsuComponent.computeStoreSrdStart(self, kernel)) for cdir in (0,1): indices = kernel["PackedC%uIndicesX"%cdir] @@ -7579,8 +7457,10 @@ def globalWriteWorkGroupInit(self, kernel): if kernel["BufferStore"]: module.add(self.allocPostLoopSrd("D")) module.add(self.allocPostLoopSrd("C")) - module.add(self.computeStoreSrdStart(kernel, ["C", "D"], sgprBpeList=["GSULog2BpeC", "GSULog2BpeD"])) - module.add(self.undefineSgpr("GSULog2BpeC")) + sgprBpeList = ["GSULog2BpeC", "GSULog2BpeD"] if kernel["GlobalSplitU"] > 0 else [] + module.add(self.computeStoreSrdStart(kernel, ["C", "D"], sgprBpeList=sgprBpeList)) + if kernel["GlobalSplitU"] > 0: + module.add(self.undefineSgpr("GSULog2BpeC")) return module ############################################################################## @@ -7666,9 +7546,10 @@ def localSplitUGlobalWriteIndices(self, kernel): src=sgpr("AddressC+1"), \ comment="sgpr -> vgpr")) - gsuLabel = Label(label=self.labels.getNameInc("GSU"), comment="") - module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) - module.add(SCBranchSCC0(labelName=gsuLabel.getLabelName(), comment="branch if GSU != 1")) + if kernel["GlobalSplitU"] > 0: + gsuLabel = Label(label=self.labels.getNameInc("GSU"), comment="") + module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) + module.add(SCBranchSCC0(labelName=gsuLabel.getLabelName(), comment="branch if GSU != 1")) if kernel["ProblemType"]["UseE"]: self.vgprs.addrE = self.vgprPool.checkOut(2, 'addrE') module.add(VMovB32( \ @@ -7699,7 +7580,8 @@ def localSplitUGlobalWriteIndices(self, kernel): dst=vgpr(self.vgprs.addrScaleAlphaVec+1), \ src=sgpr("AddressScaleAlphaVec+1"), \ comment="sgpr -> vgpr")) - module.add(gsuLabel) + if kernel["GlobalSplitU"] > 0: + module.add(gsuLabel) return module @@ -7752,9 +7634,10 @@ def notLocalSplitUGlobalWriteIndices(self, kernel): src=sgpr("AddressC+1"), \ comment="sgpr -> vgpr")) - gsuLabel = Label(label=self.labels.getNameInc("GSU"), comment="") - module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) - module.add(SCBranchSCC0(labelName=gsuLabel.getLabelName(), comment="branch if GSU != 1")) + if kernel["GlobalSplitU"] > 0: + gsuLabel = Label(label=self.labels.getNameInc("GSU"), comment="") + module.add(SCmpEQU32(src0=sgpr("GSU"), src1=1, comment="GSU == 1 ?")) + module.add(SCBranchSCC0(labelName=gsuLabel.getLabelName(), comment="branch if GSU != 1")) if kernel["ProblemType"]["UseE"]: self.vgprs.addrE = self.vgprPool.checkOut(2, 'addrE') module.add(VMovB32( \ @@ -7785,7 +7668,8 @@ def notLocalSplitUGlobalWriteIndices(self, kernel): dst=vgpr(self.vgprs.addrScaleAlphaVec+1), \ src=sgpr("AddressScaleAlphaVec+1"), \ comment="sgpr -> vgpr")) - module.add(gsuLabel) + if kernel["GlobalSplitU"] > 0: + module.add(gsuLabel) return module ############################################################################## @@ -8197,8 +8081,9 @@ def notLocalSplitUGlobalWrite(self, kernel, tPA, tPB): + noGSUBranch = (kernel["GlobalSplitU"] == 0) module = Module("notLocalSplitUGlobalWrite") - module.add(self.globalWriteElements(kernel, tPA, tPB, vectorWidths, vectorWidths_1, elements, elements_1)) + module.add(self.globalWriteElements(kernel, tPA, tPB, vectorWidths, vectorWidths_1, elements, elements_1, noGSUBranch=noGSUBranch)) self.cleanupGlobalWrite(kernel) @@ -8240,8 +8125,9 @@ def localSplitUGlobalWrite(self, kernel, tPA, tPB): vectorWidths = [fullVw, edgeVw] vectorWidths_1 = [fullVw_1, edgeVw_1] + noGSUBranch = (kernel["GlobalSplitU"] == 0) module = Module("localSplitUGlobalWrite") - module.add(self.globalWriteElements(kernel, tPA, tPB, vectorWidths, vectorWidths_1, elements_f0, elements_f1)) + module.add(self.globalWriteElements(kernel, tPA, tPB, vectorWidths, vectorWidths_1, elements_f0, elements_f1, noGSUBranch=noGSUBranch)) self.cleanupGlobalWrite(kernel) self.vgprPool.checkIn(self.accVgprLdsReduction) return module @@ -8702,7 +8588,10 @@ def globalWriteElements(self, kernel, tPA, tPB, vectorWidths_2, vectorWidths_1, # write possibilities and labels # if beta/edge combo not specified fall back to global param definition if betas is None: - hasBeta = kernel["ProblemType"]["UseBeta"] and (kernel["_GlobalAccumulation"] != 'MultipleBuffer' or (kernel["GlobalSplitUAlgorithm"] == 'MultipleBufferSingleKernel')) + hasBeta = kernel["ProblemType"]["UseBeta"] and \ + (kernel["_GlobalAccumulation"] != 'MultipleBuffer' or \ + kernel["GlobalSplitUAlgorithm"] == 'MultipleBufferSingleKernel' or \ + kernel["GlobalSplitU"] == 0) betas = [False, True] if hasBeta else [False] if edges is None: edges = [False, True] if self.do["EdgeWrite"] else [False] diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index 66926f56c4..a8636fe243 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -3691,7 +3691,7 @@ def getKeyNoInternalArgs(state): if globalParameters["SplitGSU"]: state_copy["GlobalSplitU"] = "M" if (state_copy["GlobalSplitU"] > 1) else state_copy["GlobalSplitU"] - else: + elif state["GlobalSplitU"] > 0: state_copy["GlobalSplitU"] = "M" state_copy["WorkGroupMapping"] = "M" state_copy["StaggerU"] = "M" @@ -3744,7 +3744,7 @@ def getNameMin(state, requiredParameters, ignoreInternalArgs = False): if ignoreInternalArgs: if globalParameters["SplitGSU"]: state["GlobalSplitU"] = "M" if (state["GlobalSplitU"] > 1) else state["GlobalSplitU"] - else: + elif state["GlobalSplitU"] > 0: requiredParameters["GlobalSplitU"] = False requiredParameters["WorkGroupMapping"] = False requiredParameters["StaggerU"] = False diff --git a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp index fbaba6413e..026308318e 100644 --- a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp +++ b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp @@ -873,7 +873,8 @@ namespace Tensile uint32_t gsu = problem.getParams().gsu() > 0 ? problem.getParams().gsu() : sizeMapping.globalSplitU; - rv.numWorkGroups.y *= gsu; + if(gsu > 0) + rv.numWorkGroups.y *= gsu; size_t cuCount = 0; size_t skGrid = 0; @@ -1035,7 +1036,8 @@ namespace Tensile uint32_t gsu = problem.getParams().gsu() > 0 ? problem.getParams().gsu() : sizeMapping.globalSplitU; - numWorkGroups.y *= gsu; + if(gsu > 0) + numWorkGroups.y *= gsu; numWorkItems.x += (workGroupSize.x * numWorkGroups.x * workGroupSize.y * numWorkGroups.y * workGroupSize.z * numWorkGroups.z); @@ -1094,9 +1096,6 @@ namespace Tensile singleCallArgs( problem, inputs.grouped[idx], workspaceOffsetInByte, nullptr, h_args); - uint32_t gsu = problem.getParams().gsu() > 0 ? problem.getParams().gsu() - : sizeMapping.globalSplitU; - if(sizeMapping.globalAccumulation == 3) { h_args.template append("dstD", inputs.grouped[idx].d); @@ -1194,9 +1193,6 @@ namespace Tensile rv.numWorkItems.y = rv.workGroupSize.y * rv.numWorkGroups.y; rv.numWorkItems.z = rv.workGroupSize.z * rv.numWorkGroups.z; - uint32_t gsu - = problem.getParams().gsu() > 0 ? problem.getParams().gsu() : sizeMapping.globalSplitU; - if(sizeMapping.globalAccumulation) rv.args.append("WS", inputs.ws); else if(problemType.stridedBatched) @@ -1303,8 +1299,6 @@ namespace Tensile std::string ContractionSolution::betaOnlyKernelName(Problem const& problem) const { - uint32_t gsu - = problem.getParams().gsu() > 0 ? problem.getParams().gsu() : sizeMapping.globalSplitU; std::string name = concatenate( "C", problem.cNames(), "_", DataTypeInfo::Get(problem.d().dataType()).abbrev); diff --git a/tensilelite/Tensile/Tests/common/gsu/f32_gsu.yaml b/tensilelite/Tensile/Tests/common/gsu/f32_gsu.yaml new file mode 100644 index 0000000000..7c296f842d --- /dev/null +++ b/tensilelite/Tensile/Tests/common/gsu/f32_gsu.yaml @@ -0,0 +1,58 @@ + +GlobalParameters: + NumElementsToValidate: -1 + BoundsCheck: False + KernelTime: False + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + DataInitTypeA: 12 + DataInitTypeB: 13 + DataInitTypeC: 12 + MaxWorkspaceSize: 134217728 + NumWarmups: 0 + EnqueuesPerSync: 1 + +BenchmarkProblems: + - # sgemm NT + - # ProblemType + OperationType: GEMM + DataType: s + TransposeA: False + TransposeB: True + UseBeta: True + Batched: True + + - # BenchmarkProblemSizeGroup - Standard + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 8 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [4] + - GlobalReadVectorWidthB: [4] + - GlobalSplitU: [0, 1, 2] + - LocalReadVectorWidth: [1] + - MatrixInstruction: + - [16, 16, 4, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchLocalRead: [1] + - PrefetchGlobalRead: [2] + # - StaggerU: [0, 32] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + - StoreVectorWidth: [4] + - TransposeLDS: [0] + - VectorWidthA: [4] + - VectorWidthB: [4] + - WorkGroupMapping: [1, 2, 4, 8] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [4096, 4096, 1, 4096] From b92f313c17c022fb8d80719ba2da80141f825411 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Mon, 24 Jun 2024 18:08:28 +0000 Subject: [PATCH 08/36] UpdatedSolutionStructs, reject cases and golbal accumulation code --- tensilelite/Tensile/Common.py | 36 +++++++++++---------- tensilelite/Tensile/SolutionStructs.py | 43 ++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index 5347442244..88568fb832 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -870,29 +870,33 @@ def getArchitectureName(gfxName): # Only support for kernel whose totalVgpr counts less than 256 and gcn that has control bit ACC_CD. "MIArchVgpr": [False, True], - # StreamK kernels divide work evenly among CUs by splitting along MT and K dimensions - # Total work units are calculated as (#MTs x #LoopIters) and divided among workgroups + # StreamK (SK) kernels divide work evenly among CUs by splitting along MT and K dimensions. + # Total work units are calculated as (#MTs x #LoopIters) and divided among workgroups. # In most cases each workgroup will calculate a partial tile that are accumulated in a fixup step in the same kernel - # 0: Standard data-parallel kernel - # 1: Basic StreamK - # 2: Two-Tile StreamK (each WG completes an even number of sk iterations, followed by an even number of dp tiles) - # 3: Two-Tile StreamK with DP before SK tiles + # 0 : Standard data-parallel kernel + # 1 : Basic StreamK + # 2 : Two-Tile StreamK (each WG completes an even number of sk iterations, followed by an even number of dp tiles) + # 3 : Two-Tile StreamK with DP before SK tiles # StreamK kernels can adjust the number of CUs being used. # Using fewer sometimes increases overall throughput by allowing other kernels to run in parallel. # StreamK grid is controlled by setting these enviornment variables: - # TENSILE_STREAMK_DYNAMIC_GRID enables dynamic grid mode, which automatically limits the number of CUs used for small - # problems to a subset based on the number of output tiles. - # 0 = off (default) - # 1 = on - # 2 = also reduce CUs used for large sizes to improve data-parallel portion and reduce power - # 3 = analytical grid size predictor + # TENSILE_STREAMK_FIXED_GRID lets you override the default grid size with a specific number + # 0 = override disabled (default) + # TENSILE_STREAMK_FULL_TILES sets the number of full tiles to be included in stream-k work + # -1 = use prediction model for best performance (default) + # 0 = only remainder tiles run in stream-k + # 1+ = remainder + 1 (or more) full grids of tiles run in stream-k + # TENSILE_STREAMK_DYNAMIC_GRID enables dynamic grid mode, which automatically limits the number of CUs used: + # 0 = Off, use all CUs (default) + # 1 = Only reduce CUs for small problems to number of output tiles when num_tiles < CU count. + # 2 = Also reduce CUs used for large sizes to improve data-parallel portion and reduce power. + # 3 = Analytically predict the best grid-size by weighing the cost of the fix-up step and the cost of processing MACs. # TENSILE_STREAMK_MAX_CUS allows the user to manually set maximum number of CUs used, which could free up some CUs for # other operations to run in parallel with gemm. - # 0 = use all CUs (default) # TENSILE_STREAMK_GRID_MULTIPLIER lets you set how many workgroups are created per CU being used. - # 1 = 1 WG per CU (default) - # TENSILE_STREAMK_FIXED_GRID lets you override the default grid size with a specific number - # 0 = override disabled (default) + # 1 = 1 WG per CU (default), for example. 2 will launch WGs = 2 x CU count. + # The priority of these environment variables is defined as follows: + # TENSILE_STREAMK_FIXED_GRID > TENSILE_STREAMK_DYNAMIC_GRID > TENSILE_STREAMK_MAX_CUS > TENSILE_STREAMK_GRID_MULTIPLIER "StreamK": [0, 1, 2, 3], # Determines if StreamK kernel uses atomics # 0: uses workspace to store partial tiles, accumulate in deterministic fix-up step diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index a8636fe243..f4260b6f0e 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -1110,10 +1110,10 @@ def initHelperKernelObjects(self): self.initReductionKernelObjects() ######################################## - # create BetaONly Kernels + # create BetaOnly Kernels def initBetaOnlyKernelObjects(self): self.betaOnlyKernelObjects = [] - if self["GlobalSplitU"] > 1: + if self["GlobalSplitU"] > 1 or (self["StreamK"] > 0 and self["StreamKAtomic"] == 1): if self["ProblemType"]["UseBias"]: for btype in self["ProblemType"]["BiasDataTypeList"]: state = {} @@ -1920,7 +1920,11 @@ def assignDerivedParameters(state): computeBytes = state["ProblemType"]["ComputeDataType"].numBytes() state["_GlobalAccumulation"] = None computeName = state["ProblemType"]["ComputeDataType"].toName() - if state["GlobalSplitUAlgorithm"] == 'SingleBuffer': + if state["StreamK"] > 0 and state["StreamKAtomic"] == 0: + # StreamK Workspace size + computeBytes = state["ProblemType"]["ComputeDataType"].numBytes() + state["_GlobalAccumulation"] = 'PartialsBuffer' + elif state["GlobalSplitUAlgorithm"] == 'SingleBuffer': if computeName != state["ProblemType"]["DestDataType"].toName(): state["_GlobalAccumulation"] = 'SingleBuffer' elif state["GlobalSplitUAlgorithm"] == 'MultipleBuffer': @@ -1936,6 +1940,32 @@ def assignDerivedParameters(state): state["SynchronizerSizeCheck"] = 1 # state["BatchSizeEqual"] = 1 + if state["StreamK"] != 0: + if state["MIWaveGroup"][0] * state["MIWaveGroup"][1] != 4: + reject(state, "Stream-K requries MIWaveGroup0*MIWaveGroup1=4") + if state["EnableMatrixInstruction"] and globalParameters["AsmCaps"][isa]["HasWMMA"]: + reject(state, "Stream-K untested with WMMA") + if state["GlobalSplitU"] > 0: + reject(state, "Cannot enable both Stream-K and GSU") + # if state["PersistentKernel"]: + # reject(state, "Cannot enable both Stream-K and PersistentKernel") + if not state["ProblemType"]["StridedBatched"]: + reject(state, "General batch not supported with Stream-K") + if state["ProblemType"]["GroupedGemm"]: + reject(state, "Grouped gemm not yet supported with Stream-K") + if state["StreamKAtomic"] == 1: + if not state["ProblemType"]["DataType"].isSingle(): + reject(state, "Atomic Stream-K currently only tested for SGEMM") + if not state["BufferStore"]: + reject(state, "Atomic Stream-K requires BufferStore") + if state["LocalSplitU"] > 1: + reject(state, "Atomic Stream-K not working with LocalSplitU") + else: + # If not using StreamK, clear other stream-k settings to avoid duplicate kernels + state["StreamKAtomic"] = 0 + state["StreamKXCCMapping"] = 0 + state["DebugStreamK"] = 0 + computeBytes = state["ProblemType"]["ComputeDataType"].numBytes() state["_WorkspaceSizePerElemC"] = computeBytes state["_WorkspaceSizePerElemBias"] = 0 @@ -2730,10 +2760,11 @@ def calcOptGRVW(lrvw: int, unrollMajorLDS: bool, datatype: DataType) -> int: reject(state, "Source KernelLanguage only supports LdsPadA == LdsPadB") return - # NoTailLoop parameter initialization. Set True for the following cases - # 1. ASEM is multiple of DepthU. TailLoop code will not be used in this case. + # NoTailLoop parameter initialization. + # If ASEM is multiple of DepthU TailLoop will not be used. + # Unless kernel is Stream-K; Stream-K always requires TailLoop to handle work division. state["NoTailLoop"] = False - if state["AssertSummationElementMultiple"] % state["DepthU"] == 0: + if state["AssertSummationElementMultiple"] % state["DepthU"] == 0 and state["StreamK"] == 0: state["NoTailLoop"] = True ######################################## From ca5221dc8689c1e8684aac112ef7fd07540b7adb Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Wed, 26 Jun 2024 20:07:30 +0000 Subject: [PATCH 09/36] Stream-k code gen work in progress --- .../Tensile/Components/PersistentLoop.py | 59 ++ tensilelite/Tensile/Components/StreamK.py | 620 ++++++++++++++++++ tensilelite/Tensile/Components/__init__.py | 4 +- tensilelite/Tensile/KernelWriter.py | 28 +- tensilelite/Tensile/KernelWriterAssembly.py | 64 +- .../TensileInstructions/Instructions.py | 9 +- 6 files changed, 751 insertions(+), 33 deletions(-) create mode 100644 tensilelite/Tensile/Components/PersistentLoop.py create mode 100644 tensilelite/Tensile/Components/StreamK.py diff --git a/tensilelite/Tensile/Components/PersistentLoop.py b/tensilelite/Tensile/Components/PersistentLoop.py new file mode 100644 index 0000000000..6064632df3 --- /dev/null +++ b/tensilelite/Tensile/Components/PersistentLoop.py @@ -0,0 +1,59 @@ +################################################################################ +# +# Copyright (C) 2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import Module, Label +from ..Component import Component +import abc + +class PersistentLoop(Component): + """ + Persistent loop code. + """ + @abc.abstractmethod + def openPersistentLoop(self, writer, kernel): + pass + +class PersistentLoopOff(PersistentLoop): + kernel = {"StreamK": 0} + + def openPersistentLoop(self, writer, kernel): + module = Module("PersistentLoop Off openPersistentLoop") + return module + +class PersistentLoopOn(PersistentLoop): + + @classmethod + def matches(cls, writer, debug=False): + return writer.states.kernel["StreamK"] > 0 + + def openPersistentLoop(self, writer, kernel): + module = Module("PersistentLoop On openPersistentLoop") + + # Label start of persistent loop + module.addComment2("Persistent Loop Start") + persistentLabel = Label(label="PersistentLoopStart", comment="") + module.add(persistentLabel) + + # TODO remove? + # kStr += inst("s_add_u32", sgpr("PersistentLoopIter"), sgpr("PersistentLoopIter"), hex(1), "Inc PersistentLoop Iter") # Back-up: not needed now + #kStr += str(Code.WaitCnt(self.version, 0,0,"wait for outstanding stores")) + return module diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py new file mode 100644 index 0000000000..a48e529977 --- /dev/null +++ b/tensilelite/Tensile/Components/StreamK.py @@ -0,0 +1,620 @@ +################################################################################ +# +# Copyright (C) 2024 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- +# ies of the Software, and to permit persons to whom the Software is furnished +# to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- +# PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- +# CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +################################################################################ + +from ..TensileInstructions import Module, Label, SAddU32, RegisterPoolResource, sgpr, scalarStaticDivideAndRemainder, \ + SCmpLtU32, SCSelectB32, sMagicDivAlg2, SMulI32, SSubU32, SMinU32, SMovB32, SCBranchSCC1, SCmpLeU32, VMovB32, vgpr, \ + SAddCU32, SCmpGtU32, SCMovB32, SAddI32, SCmpEqU32 +from ..Component import Component +import abc + +class XCCMapping(Component): + """ + XCC mapping code. + """ + pass + +class XCCMappingOff(XCCMapping): + kernel = {"StreamKXCCMapping": 0} + + def __call__(self, writer, kernel): + module = Module("XCCMapping Off") + return module + +class XCCMappingOn(XCCMapping): + + @classmethod + def matches(cls, writer, debug=False): + return writer.states.kernel["StreamKXCCMapping"] > 0 + + def __call__(self, writer, kernel): + module = Module("XCCMapping On") + + with writer.allocTmpSgpr(4) as tmpSgprRes: + sXCC = tmpSgprRes.idx + sGridC = tmpSgprRes.idx + 1 + sGridF = tmpSgprRes.idx + 2 + sGridM = tmpSgprRes.idx + 3 + sTmp = None + sTmpRes = None + sqTmp = None + divisor = kernel["StreamKXCCMapping"] + if ((divisor & (divisor - 1)) != 0): # Need temp registers if not power of 2 + sTmp = writer.sgprPool.checkOut(2, "sTmp", preventOverflow=False) + sTmpRes = RegisterPoolResource(idx=sTmp, size=2) + sqTmp = writer.sgprPool.checkOut(1, "sqTmp", preventOverflow=False) + + # sGridC = ceil(grid / xccm) + module.add(SAddU32(dst=sgpr(sXCC), src0=sgpr("skGrid"), src1=hex(kernel["StreamKXCCMapping"] - 1), comment="ceil(grid/xccm)")) + module.add(scalarStaticDivideAndRemainder(qReg=sgpr(sGridC), rReg=None, dReg=sgpr(sGridC), divisor=kernel["StreamKXCCMapping"], tmpSgprRes=sTmpRes, doRemainder=0)) + # sGridF = floor(grid / xccm) + # sGridM = grid % xccm + module.add(scalarStaticDivideAndRemainder(qReg=sgpr(sGridF), rReg=sgpr(sGridM), dReg=sgpr("skGrid"), divisor=kernel["StreamKXCCMapping"], tmpSgprRes=sTmpRes)) + # sXCC = wg0 % xccm + # sqtmp is temp register for quotient for non-power-of-2 case + # sqtmp overlaps temp registers, works in this case and output is discarded + module.add(scalarStaticDivideAndRemainder(qReg=sgpr(sqTmp), rReg=sgpr(sXCC), dReg=sgpr("WorkGroup0"), divisor=kernel["StreamKXCCMapping"], tmpSgprRes=sTmpRes, doRemainder=2)) + # Check if current XCC requires a remainder WG or not + module.add(SCmpLtU32(src0=sgpr(sXCC), src1=sgpr(sGridM), comment="XCCM < Remainder")) + module.add(SCSelectB32(dst=sgpr(sGridC), src0=sgpr(sGridC), src1=sgpr(sGridF), comment="Select multiplier")) + module.add(SCSelectB32(dst=sgpr(sGridM), src0=0, src1=sgpr(sGridM), comment="Select remainder")) + # WG = floor(wg0 / xccm) * xccm + XCCoffset + optional remainder + module.add(scalarStaticDivideAndRemainder(qReg=sgpr("WorkGroup0"), rReg=None, dReg=sgpr("WorkGroup0"), divisor=kernel["StreamKXCCMapping"], tmpSgprRes=sTmpRes, doRemainder=0)) + module.add(SMulI32(dst=sgpr(sXCC), src0=sgpr(sXCC), src1=sgpr(sGridC), comment="XCC group id")) + module.add(SAddU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=sgpr(sXCC), comment="Add XCC group offset")) + module.add(SAddU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=sgpr(sGridM), comment="Add remainder offset")) + if sTmp is not None: + writer.sgprPool.checkIn(sTmp) + writer.sgprPool.checkIn(sqTmp) + + return module + + +class StreamK(Component): + """ + StreamK code. + """ + + @abc.abstractmethod + def preLoop(self, writer, kernel): + pass + + @abc.abstractmethod + def graWorkGroup(self, writer, kernel): + pass + + def skTileIndex(self, writer, kernel, sTmp): + module = Module("StreamK skTileIndex") + + # Always reset pointers to handle odd-exit case which moves LRO to the upper bank + if kernel["PrefetchGlobalRead"]: # not self.prefetchAcrossPersistent + module.add(writer.localReadResetOffsets(kernel, writer.tPA)) + module.add(writer.localReadResetOffsets(kernel, writer.tPB)) + + module.addComment0("StreamK calculate tile idx and map to WG") + + # sTmp = tile index + module.add(sMagicDivAlg2(sTmp, sgpr("StreamKIter"), sgpr("MagicNumberItersPerTile"), sgpr("MagicShiftItersPerTile"))) + # sTmp+1 = tile start + module.add(SMulI32(dst=sgpr(sTmp+1), src0=sgpr(sTmp), src1=sgpr("ItersPerTile"), comment="Tile start iteration")) + # sTmp+2 = tile end + module.add(SAddU32(dst=sgpr(sTmp+2), src0=sgpr(sTmp+1), src1=sgpr("ItersPerTile"), comment="Tile end iteration")) + # local start + module.add(SSubU32(dst=sgpr("StreamKLocalStart"), src0=sgpr("StreamKIter"), src1=sgpr(sTmp+1), comment="Local iteration start")) + # local end (SK tile) + module.add(SMinU32(dst=sgpr("StreamKLocalEnd"), src0=sgpr("StreamKIterEnd"), src1=sgpr(sTmp+2), comment="1. (Local) iteration end (SK tile)")) + module.add(SSubU32(dst=sgpr("StreamKLocalEnd"), src0=sgpr("StreamKLocalEnd"), src1=sgpr(sTmp+1), comment="2. Local iteration end (SK tile)")) + + return module + + def skIndexToWG(self, writer, kernel, sTmp): + module = Module("StreamK skIndexToWG") + + # Map StreamK tile index to wg0/1 + module.addComment0("Map StreamK tile index to wg0/1/2") + module.add(sMagicDivAlg2(sTmp+1, sgpr(sTmp), sgpr("MagicNumProblemNumGroupTiles0By1"), sgpr("MagicShiftProblemNumGroupTiles0By1"))) + module.add(SMovB32(dst=sgpr("WorkGroup2"), src=sgpr(sTmp+1), comment="wg2 = Tile Idx / problemNumGroupTiles0By1")) + module.add(SMulI32(dst=sgpr(sTmp+1), src0=sgpr(sTmp+1), src1=sgpr("NumWorkGroups0"), comment="remainder part 1 : quotient * divisor")) + module.add(SMulI32(dst=sgpr(sTmp+1), src0=sgpr(sTmp+1), src1=sgpr("NumWorkGroups1"), comment="remainder part 1 : quotient * divisor")) + module.add(SSubU32(dst=sgpr(sTmp), src0=sgpr(sTmp), src1=sgpr(sTmp+1), comment="remainder")) + module.add(sMagicDivAlg2(sTmp+1, sgpr(sTmp), sgpr("MagicNumberProblemNumGroupTiles0"), sgpr("MagicShiftProblemNumGroupTiles0"))) + module.add(SMovB32(dst=sgpr("WorkGroup1"), src=sgpr(sTmp+1), comment="wg1 = Tile Idx / problemNumGroupTiles0")) + module.add(SMulI32(dst=sgpr("WorkGroup0"), src0=sgpr(sTmp+1), src1=sgpr("NumWorkGroups0"), comment="remainder part 1 : quotient * divisor")) + module.add(SSubU32(dst=sgpr("WorkGroup0"), src0=sgpr(sTmp), src1=sgpr("WorkGroup0"), comment="wg0 = Tile Idx % problemNumGroupTiles0")) + module.addSpaceLine() + + return module + + @abc.abstractmethod + def computeLoadSrd(self, writer, kernel, tc, sTmp): + pass + + def computeLoadSrdCommon(self, writer, kernel, tc, sTmp): + module = Module("StreamK Common computeLoadSrd") + + # StreamK partial tile - offset to tile start index + module.add(SMulI32(dst=sgpr(sTmp), src0=sgpr("StreamKLocalStart"), src1="DepthU", comment="StreamK tile start offset")) + strideL = writer.strideRef(tc, kernel["ProblemType"]["IndicesSummation"][0]) + module.add(writer.s_mul_u64_u32(sgpr(sTmp), sgpr(sTmp+1), sgpr(sTmp), strideL, "StreamK tile start offset")) + # Overflow check removed + # if kernel["CheckDimOverflow"] >=2: + # kStr += self.assert_eq(sgpr(sTmp+1),0) + module.add(SAddU32(dst=sgpr(tileStart+0), src0=sgpr(tileStart+0), src1=sgpr(sTmp+0), comment="accum GsuOffset term to tilestart")) + module.add(SAddCU32(dst=sgpr(tileStart+1), src0=sgpr(tileStart+1), src1=sgpr(sTmp+1), comment="accum GsuOffset term to tilestart")) + + return module + + @abc.abstractmethod + def graAddresses(self, writer, kernel, tP, vTmp): + pass + + def graAddressesCommon(self, writer, kernel, tP, vTmp): + module = Module("StreamK Common graAddresses") + + tc = tP["tensorChar"] + # StreamK partial tile - offset to tile start index + tmpOffset = writer.sgprPool.checkOut(2, "skStartOffset", preventOverflow=0) + module.add(SMulI32(dst=sgpr(tmpOffset), src0=sgpr("StreamKLocalStart"), src1="DepthU*%d" % (tP["bpe"]), comment="StreamK tile start offset")) + strideL = writer.strideRef(tc, kernel["ProblemType"]["IndicesSummation"][0]) + module.add(writer.s_mul_u64_u32(sgpr(tmpOffset), sgpr(tmpOffset+1), sgpr(tmpOffset), strideL, "StreamK tile start offset")) + # Overflow check removed + # if kernel["CheckDimOverflow"] >=2: + # kStr += self.assert_eq(sgpr(tmpOffset+1),0) + module.add(SAddU32(dst=sgpr(tmpOffset+0), src0=sgpr(tmpOffset+0), src1=sgpr("Address%s+0" % tc), comment="accum skOffset term to tilestart")) + module.add(SAddCU32(dst=sgpr(tmpOffset+1), src0=sgpr(tmpOffset+1), src1=sgpr("Address%s+1" % tc), comment="accum skOffset term to tilestart")) + module.add(VMovB32(dst=vgpr(vTmp+0), src=sgpr(tmpOffset+0))) + module.add(VMovB32(dst=vgpr(vTmp+1), src=sgpr(tmpOffset+1))) + writer.sgprPool.checkIn(tmpOffset) + + return module + + @abc.abstractmethod + def declareStaggerParms(self, writer, kernel): + pass + + def declareStaggerParmsCommon(self, writer, kernel): + module = Module("StreamK Common declareStaggerParms") + + # Set stagger=0 for partial tiles to avoid using stagger larger than workload + module.add(SCmpGtU32(src0=sgpr("StreamKLocalStart"), src1=0, comment="does wg start tile?")) + module.add(SCMovB32(dst=sgpr("StaggerUIter"), src=0, comment="set stagger=0 for partial tiles")) + module.add(SCmpLtU32(src0=sgpr("StreamKLocalEnd"), src1=sgpr("ItersPerTile"), comment="does wg finish tile?")) + module.add(SCMovB32(dst=sgpr("StaggerUIter"), src=0, comment="set stagger=0 for partial tiles")) + + return module + + @abc.abstractmethod + def tailLoopNumIter(self, writer, kernel, loopCounter): + pass + + def tailLoopNumIterCommon(self, writer, kernel, loopCounter): + module = Module("StreamK Common tailLoopNumIter") + + # skip tail loop if StreamK WG not processing final iteration + # Check if tile finished + module.add(SCmpLtU32(src0=sgpr("StreamKLocalEnd"), src1=sgpr("ItersPerTile"), comment="Check if WG processes final iteration of tile")) + module.add(SCMovB32(dst=loopCounter, src=hex(0), comment="This WG not completing tile")) + + return module + + @abc.abstractmethod + def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgprInfo): + pass + + def calculateLoopNumIterCommon(self, writer, kernel, loopCounterName, loopIdx, tmpSgprInfo): + module = Module("StreamK Common calculateLoopNumIter") + + # Use StreamK params for loop count + module.add(SSubU32(dst=sgpr(loopCounterName), src0=sgpr("StreamKLocalEnd"), src1=sgpr("StreamKLocalStart"), common="StreamK loop counter = localEnd - localStart")) + # Adjust loop count for tail loop + if not kernel["NoTailLoop"]: + tmpSgpr = tmpSgprInfo.idx + unrollIdx = writer.states.unrollIdx + loopChar = writer.states.indexChars[kernel["ProblemType"]["IndicesSummation"][unrollIdx]] + + assert kernel["DepthU"] % 2 == 0 # Assuming DepthU is power of 2, if odd DepthU were supported this divide would need 2 more temp registers for divide + module.add(scalarStaticDivideAndRemainder(qReg=sgpr(tmpSgpr), rReg=sgpr(tmpSgpr+1), dReg=sgpr("SizesSum+%u" % unrollIdx), divisor=kernel["DepthU"], tmpSgprRes=None, doRemainder=2)) + module.add(SCmpEqU32(src0=sgpr(tmpSgpr+1), src1=hex(0), comment="numIter%s == 0"%loopChar )) + module.add(SCSelectB32(dst=sgpr(tmpSgpr), src0=0, src1=1, comment="check if size uses tail loop")) + module.add(SCmpEqU32(src0=sgpr("StreamKLocalEnd"), src1=sgpr("ItersPerTile"), comment="Check if WG processes final iteration of tile")) + module.add(SCSelectB32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=0, comment="this WG runs tail loop")) + module.add(SSubU32(dst=sgpr(loopCounterName), src0=sgpr(loopCounterName), src1=sgpr(tmpSgpr), comment="Adjust loop counter for tail loop")) + + return module + + @abc.abstractmethod + def recalcLocalWriteAddresses(self, writer, kernel, tc): + pass + + def recalcLocalWriteAddressesCommon(self, writer, kernel, tc): + module = Module("StreamK Common recalcLocalWriteAddresses") + + if getattr(writer, "oriLwa%s" % tc) is None: + setattr(writer, "oriLwa%s" % tc, writer.vgprPool.checkOut(1, "OriLocalWriteddr%s" % tc)) + module.add(VMovB32(dst=vgpr(getattr(writer, "oriLwa%s" % tc)), src=vgpr("LocalWriteAddr%s" % tc), comment="back up LWA for persistent kernel + wider local read")) + + return module + + +class StreamKOff(StreamK): + kernel = {"StreamK": 0} + + def preLoop(self, writer, kernel): + module = Module("StreamK Off openLoop") + return module + + def graWorkGroup(self, writer, kernel): + module = Module("StreamK Off graWorkGroup") + return module + + def computeLoadSrd(self, writer, kernel, tc, sTmp): + module = Module("StreamK Off computeLoadSrd") + return module + + def graAddresses(self, writer, kernel, tP, vTmp): + module = Module("StreamK Off graAddresses") + + tc = tP["tensorChar"] + module.add(VMovB32(dst=vgpr(vTmp+0), src=sgpr("Address%s+0" % tc))) + module.add(VMovB32(dst=vgpr(vTmp+1), src=sgpr("Address%s+1" % tc))) + + return module + + def declareStaggerParms(self, writer, kernel): + module = Module("StreamK Off declareStaggerParms") + return module + + def tailLoopNumIter(self, writer, kernel, loopCounter): + module = Module("StreamK Off tailLoopNumIter") + return module + + def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgprInfo): + module = Module("StreamK Off calculateLoopNumIter") + + quotient = loopCounterName + dividend = "SizesSum+%u" % loopIdx #sumSize = self.sumSize(kernel, loopIdx) + divisor = kernel["DepthU"] + + if kernel["NoTailLoop"] and kernel["AssertSummationElementMultiple"] % kernel["DepthU"] != 0: + # round up SizesSum/DepthU for noTailLoop case + module.add(SAddI32(dst=sgpr(quotient), src0=(divisor - 1), src1=sgpr(dividend), comment="round up SizeSum / DepthU" )) + module.add(scalarStaticDivideAndRemainder(qReg=quotient, rReg=None, dReg=quotient, divisor=divisor, tmpSgprRes=tmpSgprInfo, doRemainder=0)) + else: + module.add(scalarStaticDivideAndRemainder(qReg=quotient, rReg=None, dReg=dividend, divisor=divisor, tmpSgprRes=tmpSgprInfo, doRemainder=0)) + + return module + + def recalcLocalWriteAddresses(self, writer, kernel, tc): + module = Module("StreamK Off recalcLocalWriteAddresses") + return module + + +class StreamKBasic(StreamK): + kernel = {"StreamK": 1} + + def preLoop(self, writer, kernel): + module = Module("StreamK Basic openLoop") + + xccMapping = Component.XCCMapping.find(writer) + module.add(xccMapping(writer, kernel)) + + module.add(SMovB32(dst=sgpr("StreamKIdx"), src=sgpr("WorkGroup0"), comment="Save original StreamK index")) + # Basic SK + module.add(SMulI32(dst=sgpr("StreamKIter"), src0=sgpr("StreamKIdx"), src1=sgpr("SKItersPerWG"), comment="StreamK starting iteration")) + module.add(SAddU32(dst=sgpr("StreamKIterEnd"), src0=sgpr("StreamKIter"), src1=sgpr("SKItersPerWG"), comment="StreamK ending iteration")) + module.add(SMinU32(dst=sgpr("StreamKIterEnd"), src0=sgpr("StreamKIterEnd"), src1=sgpr("TotalIters"), comment="Cap ending iter at total iters")) + module.add(SCmpLtU32(src0=sgpr("StreamKIter"), src1=sgpr("StreamKIterEnd"), comment="Make sure there's work to do")) + module.add(writer.longBranchScc0(Label("KernelEnd", ""), posNeg=1)) + module.add(writer.undefineSgpr("TotalIters")) + + return module + + def graWorkGroup(self, writer, kernel): + module = Module("StreamK Basic graWorkGroup") + + # StreamK workgroup mapping + sTmp = writer.sgprPool.checkOutAligned(4, 2, "SKMappingTemp", preventOverflow=0) + + module.add(self.skTileIndex(writer, kernel, sTmp)) + + # Increment StreamK iteration + module.add(SMovB32(dst=sgpr("StreamKIter"), src=sgpr(sTmp+2), comment="Increment StreamK Iteration")) + + module.add(self.skIndexToWG(writer, kernel, sTmp)) + + writer.sgprPool.checkIn(sTmp) + + return module + + def computeLoadSrd(self, writer, kernel, tc, sTmp): + module = Module("StreamK Basic computeLoadSrd") + module.add(self.computeLoadSrdCommon(writer, kernel, tc, sTmp)) + return module + + def graAddresses(self, writer, kernel, tP, vTmp): + module = Module("StreamK Basic graAddresses") + module.add(self.graAddressesCommon(writer, kernel, tP, vTmp)) + return module + + def declareStaggerParms(self, writer, kernel): + module = Module("StreamK Basic declareStaggerParms") + module.add(self.declareStaggerParmsCommon(writer, kernel)) + return module + + def tailLoopNumIter(self, writer, kernel, loopCounter): + module = Module("StreamK Basic tailLoopNumIter") + module.add(self.tailLoopNumIterCommon(writer, kernel, loopCounter)) + return module + + def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgprInfo): + module = Module("StreamK Basic calculateLoopNumIter") + module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) + return module + + def recalcLocalWriteAddresses(self, writer, kernel, tc): + module = Module("StreamK Basic recalcLocalWriteAddresses") + module.add(self.recalcLocalWriteAddressesCommon(writer, kernel, tc)) + return module + +class StreamKTwoTileOriginal(StreamK): + kernel = {"StreamK": 2} + + def preLoop(self, writer, kernel): + module = Module("StreamK TwoTileOriginal openLoop") + + xccMapping = Component.XCCMapping.find(writer) + module.add(xccMapping(writer, kernel)) + + module.add(SMovB32(dst=sgpr("StreamKIdx"), src=sgpr("WorkGroup0"), comment="Save original StreamK index")) + # Two-tile SK (SK first) + # iter count after all extra iters have been distributed + module.add(SMulI32(dst=sgpr("StreamKIter"), src0=sgpr("StreamKIdx"), src1=sgpr("SKItersPerWG"), comment="StreamK starting iteration (case: after extra iters)")) + module.add(SAddU32(dst=sgpr("StreamKIter"), src0=sgpr("StreamKIter"), src1=sgpr("skExtraIters"), comment="Add extra iters")) + module.add(SAddU32(dst=sgpr("StreamKIterEnd"), src0=sgpr("StreamKIter"), src1=sgpr("SKItersPerWG"), comment="StreamK ending iteration (case: after extra iters)")) + # iter count before all extra iters have been distributed + # sTmp+1 = SKItersPerWG + 1 extra iteration + sIter = writer.sgprPool.checkOut(2, "SKIter", preventOverflow=False) + module.add(SAddU32(dst=sgpr(sIter+1), src0=sgpr("SKItersPerWG"), src1=1, comment="Spread out extra iterations")) + module.add(SMulI32(dst=sgpr(sIter), src0=sgpr("StreamKIdx"), src1=sgpr(sIter+1), comment="StreamK starting iteration (case: before extra iters)")) + module.add(SAddU32(dst=sgpr(sIter+1), src0=sgpr(sIter), src1=sgpr(sIter+1), comment="StreamK ending iteration (case: before extra iters)")) + # select correct start/end iteration index + module.add(SCmpLtU32(src0=sgpr("StreamKIdx"), src1=sgpr("skExtraIters"), comment="Check if lane gets an extra iteration")) + module.add(SCSelectB32(dst=sgpr("StreamKIter"), src0=sgpr(sIter), src1=sgpr("StreamKIter"), comment="Set start iter")) + module.add(SCSelectB32(dst=sgpr("StreamKIterEnd"), src0=sgpr(sIter+1), src1=sgpr("StreamKIterEnd"), comment="Set end iter")) + writer.sgprPool.checkIn(sIter) + # clamp to end of sk iterations + # TODO maybe remove clamp, since extra iters code should guarantee total iterations match + sTmp = writer.sgprPool.checkOut(1, "TotalSKIters", preventOverflow=False) + module.add(SMulI32(dst=sgpr(sTmp), src0=sgpr("skTiles"), src1=sgpr("ItersPerTile"), comment="Total SK iters")) + module.add(SMinU32(dst=sgpr("StreamKIterEnd"), src0=sgpr("StreamKIterEnd"), src1=sgpr(sTmp), comment="Cap ending iter at total SK iters")) + writer.sgprPool.checkIn(sTmp) + # check if this WG has no work to do + module.add(SCmpLtU32(src0=sgpr("StreamKIter"), src1=sgpr("TotalIters"), comment="Make sure there's work to do")) + module.add(writer.longBranchScc0(Label("KernelEnd", ""), posNeg=1)) + + return module + + def graWorkGroup(self, writer, kernel): + module = Module("StreamK TwoTileOriginal graWorkGroup") + + # StreamK workgroup mapping + sTmp = writer.sgprPool.checkOutAligned(4, 2, "SKMappingTemp", preventOverflow=0) + + module.add(self.skTileIndex(writer, kernel, sTmp)) + + # local end (DP tile) + # TODO This line isnt needed? + module.add(SSubU32(dst=sgpr(sTmp+3), src0=sgpr(sTmp+2), src1=sgpr(sTmp+1), comment="Local iteration end (DP tile)")) + # select correct local end + module.add(SCmpLtU32(src0=sgpr("StreamKIter"), src1=sgpr("StreamKIterEnd"), comment="Check if in SK or DP section")) + module.add(SCSelectB32(dst=sgpr("StreamKLocalEnd"), src0=sgpr("StreamKLocalEnd"), src1=sgpr(sTmp+3), comment="Apply SK or DP end iteration")) + + # Increment StreamK iteration + # If moving from SK to DP, next iteration is first DP + # sTmp = offset to first DP tile + module.add(SMulI32(dst=sgpr(sTmp+3), src0=sgpr("skTiles"), src1=sgpr("ItersPerTile"), comment="Offset to first DP tile")) + module.add(SMulI32(dst=sgpr(sTmp+1), src0=sgpr("StreamKIdx"), src1=sgpr("ItersPerTile"), comment="WG tile offset")) + module.add(SAddU32(dst=sgpr(sTmp+3), src0=sgpr(sTmp+3), src1=sgpr(sTmp+1), comment="DP start offset + WG offset")) + # If already in DP, add dpShift + module.add(SMulI32(dst=sgpr(sTmp+1), src0=sgpr("skGrid"), src1=sgpr("ItersPerTile"), comment="DP iterations shift")) + module.add(SAddU32(dst=sgpr(sTmp+1), src0=sgpr(sTmp+1), src1=sgpr("StreamKIter"), comment="Add DP shift")) + # Save DP iter in sTmp + module.add(SCmpLtU32(src0=sgpr("StreamKIter"), src1=sgpr("StreamKIterEnd"), comment="Check if in SK or DP section")) + module.add(SCSelectB32(dst=sgpr(sTmp+3), src0=sgpr(sTmp+3), src1=sgpr(sTmp+1), comment="Select first DP tile, or add DP shift")) + # If staying in SK portion, next iteration is sTmp+2 + module.add(SCmpLtU32(src0=sgpr(sTmp+2), src1=sgpr("StreamKIterEnd"), comment="Check if there are more SK tiles")) + module.add(SCSelectB32(dst=sgpr("StreamKIter"), src0=sgpr(sTmp+2), src1=sgpr(sTmp+3), comment="Select next SK or DP tile")) + + module.add(self.skIndexToWG(writer, kernel, sTmp)) + + writer.sgprPool.checkIn(sTmp) + + return module + + def computeLoadSrd(self, writer, kernel, tc, sTmp): + module = Module("StreamK TwoTileOriginal computeLoadSrd") + module.add(self.computeLoadSrdCommon(writer, kernel, tc, sTmp)) + return module + + def graAddresses(self, writer, kernel, tP, vTmp): + module = Module("StreamK TwoTileOriginal graAddresses") + module.add(self.graAddressesCommon(writer, kernel, tP, vTmp)) + return module + + def declareStaggerParms(self, writer, kernel): + module = Module("StreamK TwoTileOriginal declareStaggerParms") + module.add(self.declareStaggerParmsCommon(writer, kernel)) + return module + + def tailLoopNumIter(self, writer, kernel, loopCounter): + module = Module("StreamK TwoTileOriginal tailLoopNumIter") + module.add(self.tailLoopNumIterCommon(writer, kernel, loopCounter)) + return module + + def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgprInfo): + module = Module("StreamK TwoTileOriginal calculateLoopNumIter") + module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) + return module + + def recalcLocalWriteAddresses(self, writer, kernel, tc): + module = Module("StreamK TwoTileOriginal recalcLocalWriteAddresses") + module.add(self.recalcLocalWriteAddressesCommon(writer, kernel, tc)) + return module + +class StreamKTwoTileDPFirst(StreamK): + kernel = {"StreamK": 3} + + def preLoop(self, writer, kernel): + module = Module("StreamK TwoTileDPFirst openLoop") + + xccMapping = Component.XCCMapping.find(writer) + module.add(xccMapping(writer, kernel)) + + module.add(SMovB32(dst=sgpr("StreamKIdx"), src=sgpr("WorkGroup0"), comment="Save original StreamK index")) + # Two-tile SK (DP first) + # Do DP tiles before SK + skInitDone = Label("SK_InitDone", "") + module.add(SMulI32(dst=sgpr("StreamKIter"), src0=sgpr("StreamKIdx"), src1=sgpr("ItersPerTile"), comment="DP starting iteration (case: DP work to do)")) + module.add(SMovB32(dst=sgpr("StreamKIterEnd"), src=sgpr("TotalIters"), comment="DP ending iteration (case: only DP work to do)")) + sTmp = writer.sgprPool.checkOut(1, "TotalSKIters", preventOverflow=False) + module.add(SMulI32(dst=sgpr(sTmp), src0=sgpr("skTiles"), src1=sgpr("ItersPerTile"), comment="Total SK iters")) + module.add(SCmpLtU32(src0=sgpr(sTmp), src1=sgpr("TotalIters"), comment="Check if there are DP tiles to do")) + module.add(SCBranchSCC1(labelName=skInitDone, comment="Done init")) + writer.sgprPool.checkIn(sTmp) + + # If there are no DP tiles to do, regular SK init + # iter count after all extra iters have been distributed + module.add(SMulI32(dst=sgpr("StreamKIter"), src0=sgpr("StreamKIdx"), src1=sgpr("SKItersPerWG"), comment="StreamK starting iteration (case: after extra iters)")) + module.add(SAddU32(dst=sgpr("StreamKIter"), src0=sgpr("StreamKIter"), src1=sgpr("skExtraIters"), comment="Add extra iters")) + module.add(SAddU32(dst=sgpr("StreamKIterEnd"), src0=sgpr("StreamKIter"), src1=sgpr("SKItersPerWG"), comment="StreamK ending iteration (case: after extra iters)")) + # iter count before all extra iters have been distributed + # sTmp+1 = SKItersPerWG + 1 extra iteration + sIter = writer.sgprPool.checkOut(2, "SKIter", preventOverflow=False) + module.add(SAddU32(dst=sgpr(sIter+1), src0=sgpr("SKItersPerWG"), src1=1, comment="Spread out extra iterations")) + module.add(SMulI32(dst=sgpr(sIter), src0=sgpr("StreamKIdx"), src1=sgpr(sIter+1), comment="StreamK starting iteration (case: before extra iters)")) + module.add(SAddU32(dst=sgpr(sIter+1), src0=sgpr(sIter), src1=sgpr(sIter+1), comment="StreamK ending iteration (case: before extra iters)")) + # select correct start/end iteration index + module.add(SCmpLtU32(src0=sgpr("StreamKIdx"), src1=sgpr("skExtraIters"), comment="Check if lane gets an extra iteration")) + module.add(SCSelectB32(dst=sgpr("StreamKIter"), src0=sgpr(sIter), src1=sgpr("StreamKIter"), comment="Set start iter")) + module.add(SCSelectB32(dst=sgpr("StreamKIterEnd"), src0=sgpr(sIter+1), src1=sgpr("StreamKIterEnd"), comment="Set end iter")) + writer.sgprPool.checkIn(sIter) + # clamp to end of sk iterations + # TODO maybe remove clamp, since extra iters code should guarantee total iterations match + sTmp = writer.sgprPool.checkOut(1, "TotalSKIters", preventOverflow=False) + module.add(SMulI32(dst=sgpr(sTmp), src0=sgpr("skTiles"), src1=sgpr("ItersPerTile"), comment="Total SK iters")) + module.add(SMinU32(dst=sgpr("StreamKIterEnd"), src0=sgpr("StreamKIterEnd"), src1=sgpr(sTmp), comment="Cap ending iter at total SK iters")) + writer.sgprPool.checkIn(sTmp) + + module.add(skInitDone) + # check if this WG has no work to do + module.add(SCmpLtU32(src0=sgpr("StreamKIter"), src1=sgpr("TotalIters"), comment="Make sure there's work to do")) + module.add(writer.longBranchScc0(Label("KernelEnd", ""), posNeg=1)) + + return module + + def graWorkGroup(self, writer, kernel): + module = Module("StreamK TwoTileDPFirst graWorkGroup") + + # StreamK workgroup mapping + sTmp = writer.sgprPool.checkOutAligned(4, 2, "SKMappingTemp", preventOverflow=0) + + module.add(self.skTileIndex(writer, kernel, sTmp)) + + skUpdateDone = Label("SK_UpdateDone", "") + # sTmp+3 = Offset to first SK tile + module.add(SMulI32(dst=sgpr(sTmp+3), src0=sgpr("skTiles"), src1=sgpr("ItersPerTile"), comment="Total SK iters")) + module.add(SSubU32(dst=sgpr(sTmp+3), src0=sgpr("TotalIters"), src1=sgpr(sTmp+3), comment="Offset to first SK tile")) + # If in DP, add dpShift + module.add(SMulI32(dst=sgpr(sTmp+1), src0=sgpr("skGrid"), src1=sgpr("ItersPerTile"), comment="DP iterations shift")) + module.add(SAddU32(dst=sgpr(sTmp+1), src0=sgpr(sTmp+1), src1=sgpr("StreamKIter"), comment="Add DP shift")) + # if sTmp+1 < sTmp+3, continue DP (add dpShift) + module.add(SCmpLtU32(src0=sgpr(sTmp+1), src1=sgpr(sTmp+3), comment="Check if still in DP section")) + module.add(SCBranchSCC1(labelName=skUpdateDone, comment="Done update")) + # if StreamKIter >= sTmp+3, continue SK (add skShift?) + module.add(SMovB32(dst=sgpr(sTmp+1), src=sgpr(sTmp+2), comment="SK iterations shift")) + module.add(SCmpLeU32(src0=sgpr(sTmp+3), src1=sgpr("StreamKIter"), comment="Check if continuing in SK section")) + module.add(SCBranchSCC1(labelName=skUpdateDone, comment="Done update")) + # if sTmp+1 > sTmp+3 and StreamKIter < sTmp+3, switch from DP to SK (add dpShift) + # iter count after all extra iters have been distributed + module.add(SMulI32(dst=sgpr("StreamKIter"), src0=sgpr("StreamKIdx"), src1=sgpr("SKItersPerWG"), comment="StreamK starting iteration (case: after extra iters)")) + module.add(SAddU32(dst=sgpr("StreamKIter"), src0=sgpr("StreamKIter"), src1=sgpr("skExtraIters"), comment="Add extra iters")) + module.add(SAddU32(dst=sgpr("StreamKIterEnd"), src0=sgpr("StreamKIter"), src1=sgpr("SKItersPerWG"), comment="StreamK ending iteration (case: after extra iters)")) + # iter count before all extra iters have been distributed + # sTmp+1 = SKItersPerWG + 1 extra iteration + sIter = writer.sgprPool.checkOut(2, "SKIter", preventOverflow=0) + module.add(SAddU32(dst=sgpr(sIter+1), src0=sgpr("SKItersPerWG"), src1=1, comment="Spread out extra iterations")) + module.add(SMulI32(dst=sgpr(sIter), src0=sgpr("StreamKIdx"), src1=sgpr(sIter+1), comment="StreamK starting iteration (case: before extra iters)")) + module.add(SAddU32(dst=sgpr(sIter+1), src0=sgpr(sIter), src1=sgpr(sIter+1), comment="StreamK ending iteration (case: before extra iters)")) + # select correct start/end iteration index + module.add(SCmpLtU32(src0=sgpr("StreamKIdx"), src1=sgpr("skExtraIters"), comment="Check if lane gets an extra iteration")) + module.add(SCSelectB32(dst=sgpr("StreamKIter"), src0=sgpr(sIter), src1=sgpr("StreamKIter"), comment="Set start iter")) + module.add(SCSelectB32(dst=sgpr("StreamKIterEnd"), src0=sgpr(sIter+1), src1=sgpr("StreamKIterEnd"), comment="Set end iter")) + writer.sgprPool.checkIn(sIter) + module.add(SAddU32(dst=sgpr(sTmp+1), src0=sgpr("StreamKIter"), src1=sgpr(sTmp+3), comment="Offset to start of SK section")) + module.add(SAddU32(dst=sgpr("StreamKIterEnd"), src0=sgpr("StreamKIterEnd"), src1=sgpr(sTmp+3), comment="Offset to start of SK section")) + # clamp to end of iterations + # TODO maybe remove clamp, since extra iters code should guarantee total iterations match + module.add(SMinU32(dst=sgpr("StreamKIterEnd"), src0=sgpr("StreamKIterEnd"), src1=sgpr("TotalIters"), comment="Cap ending iter at total SK iters")) + # check if this WG has no work to do + # TODO Shouldn't need this check! + module.add(SCmpLtU32(src0=sgpr("StreamKIter"), src1=sgpr("TotalIters"), comment="Make sure there's work to do")) + module.add(writer.longBranchScc0(Label("KernelEnd", ""), posNeg=1)) # reuse tmp + + # If in SK, next iteration is sTmp+2 + # Increment StreamK iteration + module.add(skUpdateDone) + module.add(SMovB32(dst=sgpr("StreamKIter"), src=sgpr(sTmp+1), comment="Store current iteration")) + + module.add(self.skIndexToWG(writer, kernel, sTmp)) + + writer.sgprPool.checkIn(sTmp) + + return module + + def computeLoadSrd(self, writer, kernel, tc, sTmp): + module = Module("StreamK TwoTileDPFirst computeLoadSrd") + module.add(self.computeLoadSrdCommon(writer, kernel, tc, sTmp)) + return module + + def graAddresses(self, writer, kernel, tP, vTmp): + module = Module("StreamK TwoTileDPFirst graAddresses") + module.add(self.graAddressesCommon(writer, kernel, tP, vTmp)) + return module + + def declareStaggerParms(self, writer, kernel): + module = Module("StreamK TwoTileDPFirst declareStaggerParms") + module.add(self.declareStaggerParmsCommon(writer, kernel)) + return module + + def tailLoopNumIter(self, writer, kernel, loopCounter): + module = Module("StreamK TwoTileDPFirst tailLoopNumIter") + module.add(self.tailLoopNumIterCommon(writer, kernel, loopCounter)) + return module + + def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgprInfo): + module = Module("StreamK TwoTileDPFirst calculateLoopNumIter") + module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) + return module + + def recalcLocalWriteAddresses(self, writer, kernel, tc): + module = Module("StreamK TwoTileDPFirst recalcLocalWriteAddresses") + module.add(self.recalcLocalWriteAddressesCommon(writer, kernel, tc)) + return module diff --git a/tensilelite/Tensile/Components/__init__.py b/tensilelite/Tensile/Components/__init__.py index 449b92c563..fe3f952c66 100644 --- a/tensilelite/Tensile/Components/__init__.py +++ b/tensilelite/Tensile/Components/__init__.py @@ -46,5 +46,7 @@ def use(): pass "SIA", "GlobalWriteBatch", "SumUnroll", - "GSU" + "GSU", + "StreamK", + "PersistentLoop" ] diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 1f42b6febf..e6ba449f7b 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -2073,6 +2073,13 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): module.add(Label("ASM_Start", "Main body of the asm kernel")) module.add(self.defineAndResources(kernel, tensorParametersA, tensorParametersB, tPM)) + # Initialize stream-k loop + skComponent = Component.StreamK.find(self) + module.add(skComponent.preLoop(self, kernel)) + # Open persistent loop + loopComponent = Component.PersistentLoop.find(self) + module.add(loopComponent.openPersistentLoop(self, kernel)) + module.add(self.setupNewTile(kernel, tensorParametersA, tensorParametersB, isOptNLL=False)) if self.do["executeToPrefetchEnd"]: @@ -2095,7 +2102,7 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): # These cases loop back and run the prefetch loop again # we need an extra barrier to ensure that the ds_reads (either for SR or MFMA) from previous iteration # have finished before we generate the prefetch for the next summation index. - if self.states.actualSummationLoops>1: + if kernel["StreamK"] > 0 or self.states.actualSummationLoops>1: module.add(SBarrier()) # local write @@ -2848,8 +2855,10 @@ def readWriteVectors(mat, vw, kernel): self.states.bpeCinternal = int(self.states.bpr * kernel["ProblemType"]["ComputeDataType"].numRegisters()) self.states.bpeCexternalGSU1 = int(self.states.bpr * kernel["ProblemType"]["DestDataType"].numRegisters()) - self.states.bpeCexternal = self.states.bpeCinternal if kernel["_GlobalAccumulation"] else \ - self.states.bpeCexternalGSU1 + self.states.bpeCexternal = self.states.bpeCexternalGSU1 + if kernel["_GlobalAccumulation"] and kernel["_GlobalAccumulation"] != 'PartialsBuffer': + self.states.bpeCexternal = self.states.bpeCinternal + # special case for wmma h and b if (kernel["EnableMatrixInstruction"] @@ -3642,14 +3651,17 @@ def readWriteVectors(mat, vw, kernel): for key, _ in self.sgprs.items(): self.states.nonPostLoopSgpr.append(key) # Manually remove some additional unused sgpr - self.states.nonPostLoopSgpr.remove("WGM") for i in range(kernel["ProblemType"]["NumIndicesSummation"]): self.states.nonPostLoopSgpr.remove(self.loopCounterName(kernel,i)) self.states.nonPostLoopSgpr.remove("OrigLoopCounter") - self.states.nonPostLoopSgpr.remove("AddressA") - self.states.nonPostLoopSgpr.remove("AddressB") - self.states.nonPostLoopSgpr.remove("StridesA") - self.states.nonPostLoopSgpr.remove("StridesB") + + if not kernel["StreamK"]: + # Persistent loop requires arguments to remain for next tile + self.states.nonPostLoopSgpr.remove("WGM") + self.states.nonPostLoopSgpr.remove("AddressA") + self.states.nonPostLoopSgpr.remove("AddressB") + self.states.nonPostLoopSgpr.remove("StridesA") + self.states.nonPostLoopSgpr.remove("StridesB") self.states.preloadScaleA = False self.states.preloadScaleB = False diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 72d2d158f4..811bf49e65 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -388,6 +388,17 @@ def defineVariableSgprs(self, kernel): # (we reclaim them to use as temps, typically for execmasks) # Mostly impacts flat kernels and GSU edge since these need SGPR # for conditionals + + if kernel["StreamK"]: + # StreamK vars + module.add(self.defineSgpr("StreamKIdx", 1)) + module.add(self.defineSgpr("StreamKIter", 1)) + module.add(self.defineSgpr("StreamKIterEnd", 1)) + module.add(self.defineSgpr("StreamKLocalStart", 1)) + module.add(self.defineSgpr("StreamKLocalEnd", 1)) + if kernel["StreamKAtomic"] == 0: + module.add(self.defineSgpr("SrdWS", 4, 4)) + if kernel["BufferLoad"]: # resource descriptor (SRD) A and B, must be aligned on 4-SGPR boundary module.add(self.defineSgpr("SrdA", 4, 4)) @@ -1899,6 +1910,9 @@ def graWorkGroup(self, kernel): module = Module("graWorkGroup") module.addComment0("graWorkGroup mapping") + skComponent = Component.StreamK.find(self) + module.add(skComponent.graWorkGroup(self, kernel)) + gsuComponent = Component.GSU.find(self) module.add(gsuComponent.graWorkGroup(self, kernel)) @@ -2671,6 +2685,9 @@ def computeLoadSrd(self, kernel, tP, tc, indices, bpe): if not self.isConstUnitStride(strideF): module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(tileStart), sgpr(tileStart+1), sgpr(tileStart+0), \ strideF, "tlu=0, scaled tile-offset by stride")) + + skComponent = Component.StreamK.find(self) + module.add(skComponent.computeLoadSrd(self, kernel, tc, stmp)) gsuComponent = Component.GSU.find(self) module.add(gsuComponent.computeLoadSrd(self, kernel, tP, stmp, tileStart)) @@ -2819,8 +2836,10 @@ def graAddresses(self, kernel, tP): #module.add(self.getBomb(0x13)) # after addresses and SRD set else: tmp = self.vgprPool.checkOut(2, "tmp", self.states.preventVgprOverflowDuringNewTile) - module.add(VMovB32(dst=vgpr(tmp+0), src=sgpr("Address%s+0"%tP["tensorChar"]))) - module.add(VMovB32(dst=vgpr(tmp+1), src=sgpr("Address%s+1"%tP["tensorChar"]))) + + skComponent = Component.StreamK.find(self) + module.add(skComponent.graAddresses(self, kernel, tc, tmp)) + for perp in range(0, tP["nrp"]): for sPerp in range(0, tP["nrpv"]): for para in range(0, tP["nrc"]): @@ -3493,6 +3512,10 @@ def declareStaggerParms(self, kernel): comment="Compute actual stagger start for this tile")) module.add(SLShiftLeftB32(dst=sgpr("StaggerUIter"), src=sgpr("StaggerUIter"), \ shiftHex=sgpr(staggerUStrideShift), comment="shift by StaggerUStride")) + + skComponent = Component.StreamK.find(self) + module.add(skComponent.declareStaggerParms(self, kernel)) + return module ############################################################################## @@ -3716,6 +3739,9 @@ def calculateLoopNumIter(self, kernel, tPA, tPB, loopIdx): self.vgprPool.checkIn(wave_id) self.vgprPool.checkIn(tmpVgpr) + skComponent = Component.StreamK.find(self) + module.add(skComponent.tailLoopNumIter(self, kernel, loopCounter)) + gsuComponent = Component.GSU.find(self) module.add(gsuComponent.tailLoopNumIter(self, kernel, loopCounter)) @@ -3733,23 +3759,10 @@ def calculateLoopNumIter(self, kernel, tPA, tPB, loopIdx): loopCounter = sgpr(loopCounterName) if not self.do["PreLoop"]: module.add(ValueEndif()) - sumSize = "SizesSum+%u"%loopIdx - #sumSize = self.sumSize(kernel, loopIdx) - - # TODO - use named arguments with self.allocTmpSgpr(3) as tmpSgprInfo: - tmpSgpr = tmpSgprInfo.idx - quotient = loopCounterName - dividend = sumSize - divisor = kernel["DepthU"] - if kernel["NoTailLoop"] and kernel["AssertSummationElementMultiple"] % kernel["DepthU"] != 0: - # round up SizesSum/DepthU for noTailLoop case - module.add(SAddI32(dst=sgpr(quotient), src0=(divisor - 1), src1=sgpr(dividend), \ - comment="round up SizeSum / DepthU" )) - module.add(scalarStaticDivideAndRemainder(quotient, None, quotient, divisor, tmpSgprInfo, 0)) - else: - module.add(scalarStaticDivideAndRemainder(quotient, None, dividend, divisor, tmpSgprInfo, 0)) - + skComponent = Component.StreamK.find(self) + module.add(skComponent.calculateLoopNumIter(self, kernel, loopCounterName, loopIdx, tmpSgprInfo)) + gsuComponent = Component.GSU.find(self) module.add(gsuComponent.calculateLoopNumIter(self, kernel, loopCounterName, tmpSgprInfo)) @@ -4080,7 +4093,7 @@ def closeLoop(self, kernel, tPA, tPB, loopIdx, finalLoop, emitEndLabelOnly=False module.add(loopLabelEnd) if tailLoop: - if len(kernel["ProblemType"]["IndicesSummation"]) > 1: + if len(kernel["ProblemType"]["IndicesSummation"]) > 1 or kernel["StreamK"]: # recover the 'damage' done to LRO: # if LRA is backed-up before (wlr case), we simply restore the addr (sub inc*loop doesn't work) @@ -4185,6 +4198,7 @@ def endSummation(self, kernel, tPA, tPB, noSkipLoad = True, label = None): (vbegin, vbegin+vsize)) lastRegTag=None + for i in range(0, self.sgprPool.size()): regTag = self.sgprPool.pool[i].tag if regTag != lastRegTag: @@ -4897,9 +4911,12 @@ def openSumAtLeastUnroll(self, kernel, prefetch, isOptNLL): if self.do["ApplyAlpha"]: # (The new hgemm (h,h,h,h,s,s) is included in ComputeType=Single) if kernel["ProblemType"]["ComputeDataType"].isHalf(): - # for (h,h,h,h,h,h) no HPA, - module.add(SMovB32(dst=sgpr(tmpSgpr), src=hex(0x3c003c00), comment="Packed alpha==1.0")) - module.add(SCmpEQU32(src0=sgpr("Alpha"), src1=sgpr(tmpSgpr), comment="alpha == 1.0?")) + if kernel["ProblemType"]["HighPrecisionAccumulate"] and kernel["StreamK"]: + module.add(SCmpEQU32(src0=sgpr("Alpha"), src1=1.0, comment="Alpha == 1.0 ?")) + else: + # for (h,h,h,h,h,h) no HPA, + module.add(SMovB32(dst=sgpr(tmpSgpr), src=hex(0x3c003c00), comment="Packed alpha==1.0")) + module.add(SCmpEQU32(src0=sgpr("Alpha"), src1=sgpr(tmpSgpr), comment="alpha == 1.0?")) # Shouldn't go here. Currently, DataType=B->ComputeDataType=S # (bf-gemm is included in ComputeType=Single) @@ -6238,6 +6255,9 @@ def recalcLocalWriteAddresses(self, kernel, tP): [tP["localWriteStrideTile"], tP["localWriteStrideUnroll"]] ) tP["localWriteInstruction"] = self.memoryInstructions["LocalWrite"][newInstIdx] + skComponent = Component.StreamK.find(self) + module.add(skComponent.recalcLocalWriteAddresses(self, kernel)) + # local write tile assignments module.add(self.lwaTileAssignment(kernel, tP)) # local write unroll assignments diff --git a/tensilelite/Tensile/TensileInstructions/Instructions.py b/tensilelite/Tensile/TensileInstructions/Instructions.py index c47c00c279..6aa54553aa 100644 --- a/tensilelite/Tensile/TensileInstructions/Instructions.py +++ b/tensilelite/Tensile/TensileInstructions/Instructions.py @@ -1255,15 +1255,20 @@ def __init__(self, src0, src1, comment="") -> None: super().__init__(InstType.INST_I32, None, [src0, src1], None, None, comment) self.setInst("s_cmp_ge_i32") +class SCmpGeU32(CommonInstruction): + def __init__(self, src0, src1, comment="") -> None: + super().__init__(InstType.INST_U32, None, [src0, src1], None, None, comment) + self.setInst("s_cmp_ge_u32") + class SCmpGtI32(CommonInstruction): def __init__(self, src0, src1, comment="") -> None: super().__init__(InstType.INST_I32, None, [src0, src1], None, None, comment) self.setInst("s_cmp_gt_i32") -class SCmpGeU32(CommonInstruction): +class SCmpGtU32(CommonInstruction): def __init__(self, src0, src1, comment="") -> None: super().__init__(InstType.INST_U32, None, [src0, src1], None, None, comment) - self.setInst("s_cmp_ge_u32") + self.setInst("s_cmp_gt_u32") class SCmpLeI32(CommonInstruction): def __init__(self, src0, src1, comment="") -> None: From 8f5a915aed3538782522af118813270ac3ab05d1 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Wed, 26 Jun 2024 22:47:49 +0000 Subject: [PATCH 10/36] Some bug fixes and more work on persistent loop --- .../Tensile/Components/PersistentLoop.py | 51 ++++++++++- tensilelite/Tensile/Components/StreamK.py | 85 ++++++------------- tensilelite/Tensile/KernelWriter.py | 2 +- tensilelite/Tensile/KernelWriterAssembly.py | 11 ++- tensilelite/Tensile/SolutionStructs.py | 4 +- 5 files changed, 87 insertions(+), 66 deletions(-) diff --git a/tensilelite/Tensile/Components/PersistentLoop.py b/tensilelite/Tensile/Components/PersistentLoop.py index 6064632df3..cf6a474479 100644 --- a/tensilelite/Tensile/Components/PersistentLoop.py +++ b/tensilelite/Tensile/Components/PersistentLoop.py @@ -20,7 +20,7 @@ # CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ################################################################################ -from ..TensileInstructions import Module, Label +from ..TensileInstructions import Module, Label, VMovB32, vgpr from ..Component import Component import abc @@ -28,16 +28,37 @@ class PersistentLoop(Component): """ Persistent loop code. """ + def __call__(self): + assert(0) + @abc.abstractmethod def openPersistentLoop(self, writer, kernel): pass + @abc.abstractmethod + def recalcLocalWriteAddresses(self, writer, kernel, tc): + pass + + @abc.abstractmethod + def recalcLocalReadAddressesAB(self, writer, kernel): + pass + + class PersistentLoopOff(PersistentLoop): kernel = {"StreamK": 0} def openPersistentLoop(self, writer, kernel): module = Module("PersistentLoop Off openPersistentLoop") return module + + def recalcLocalWriteAddresses(self, writer, kernel, tc): + module = Module("PersistentLoop Off recalcLocalWriteAddresses") + return module + + def recalcLocalReadAddressesAB(self, writer, kernel): + module = Module("PersistentLoop Off recalcLocalReadAddressesAB") + return module + class PersistentLoopOn(PersistentLoop): @@ -57,3 +78,31 @@ def openPersistentLoop(self, writer, kernel): # kStr += inst("s_add_u32", sgpr("PersistentLoopIter"), sgpr("PersistentLoopIter"), hex(1), "Inc PersistentLoop Iter") # Back-up: not needed now #kStr += str(Code.WaitCnt(self.version, 0,0,"wait for outstanding stores")) return module + + def recalcLocalWriteAddresses(self, writer, kernel, tc): + module = Module("PersistentLoop On recalcLocalWriteAddresses") + + if getattr(writer, "oriLwa%s" % tc) is None: + setattr(writer, "oriLwa%s" % tc, writer.vgprPool.checkOut(1, "OriLocalWriteddr%s" % tc)) + module.add(VMovB32(dst=vgpr(getattr(writer, "oriLwa%s" % tc)), src=vgpr("LocalWriteAddr%s" % tc), comment="back up LWA for persistent kernel + wider local read")) + + return module + + def recalcLocalReadAddressesAB(self, writer, kernel): + module = Module("PersistentLoop On recalcLocalReadAddressesAB") + + needRecalc = writer.states.numReadsIterCoalescedA > 1 or writer.states.numReadsIterCoalescedB > 1 + # backup LocalReadAddr + # LdsPad + LBSPP case, need to backup LocalReadAddr even if recalc is not done + needBackupLRAddr = needRecalc or (kernel["LdsPadA"] and kernel["LdsBlockSizePerPadA"] or kernel["LdsPadB"] and kernel["LdsBlockSizePerPadB"]) + + if needBackupLRAddr: + # need to back-up the LRA before reCalculation for wider local read (when no wlr, no need to do this) + if writer.oriLraA is None: # and not kernel["DirectToVgprA"]: # no local read code if DirectToVgpr is enabled + writer.oriLraA = writer.vgprPool.checkOut(1, "OriLocalReadAddrA") + module.add(VMovB32(dst=vgpr(writer.oriLraA), src=vgpr("LocalReadAddrA"), comment="back up LRA for persistent kernel + wider local read")) + if writer.oriLraB is None: # and not kernel["DirectToVgprB"]: # no local read code if DirectToVgpr is enabled + writer.oriLraB = writer.vgprPool.checkOut(1, "OriLocalReadAddrB") + module.add(VMovB32(dst=vgpr(writer.oriLraB), src=vgpr("LocalReadAddrB"), comment="back up LRA for persistent kernel + wider local read")) + + return module diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index a48e529977..f5b9cf55e6 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -22,7 +22,7 @@ from ..TensileInstructions import Module, Label, SAddU32, RegisterPoolResource, sgpr, scalarStaticDivideAndRemainder, \ SCmpLtU32, SCSelectB32, sMagicDivAlg2, SMulI32, SSubU32, SMinU32, SMovB32, SCBranchSCC1, SCmpLeU32, VMovB32, vgpr, \ - SAddCU32, SCmpGtU32, SCMovB32, SAddI32, SCmpEqU32 + SAddCU32, SCmpGtU32, SCMovB32, SAddI32, SCmpEQU32 from ..Component import Component import abc @@ -64,20 +64,20 @@ def __call__(self, writer, kernel): # sGridC = ceil(grid / xccm) module.add(SAddU32(dst=sgpr(sXCC), src0=sgpr("skGrid"), src1=hex(kernel["StreamKXCCMapping"] - 1), comment="ceil(grid/xccm)")) - module.add(scalarStaticDivideAndRemainder(qReg=sgpr(sGridC), rReg=None, dReg=sgpr(sGridC), divisor=kernel["StreamKXCCMapping"], tmpSgprRes=sTmpRes, doRemainder=0)) + module.add(scalarStaticDivideAndRemainder(qReg=sGridC, rReg=None, dReg=sGridC, divisor=kernel["StreamKXCCMapping"], tmpSgprRes=sTmpRes, doRemainder=0)) # sGridF = floor(grid / xccm) # sGridM = grid % xccm - module.add(scalarStaticDivideAndRemainder(qReg=sgpr(sGridF), rReg=sgpr(sGridM), dReg=sgpr("skGrid"), divisor=kernel["StreamKXCCMapping"], tmpSgprRes=sTmpRes)) + module.add(scalarStaticDivideAndRemainder(qReg=sGridF, rReg=sGridM, dReg="skGrid", divisor=kernel["StreamKXCCMapping"], tmpSgprRes=sTmpRes)) # sXCC = wg0 % xccm # sqtmp is temp register for quotient for non-power-of-2 case # sqtmp overlaps temp registers, works in this case and output is discarded - module.add(scalarStaticDivideAndRemainder(qReg=sgpr(sqTmp), rReg=sgpr(sXCC), dReg=sgpr("WorkGroup0"), divisor=kernel["StreamKXCCMapping"], tmpSgprRes=sTmpRes, doRemainder=2)) + module.add(scalarStaticDivideAndRemainder(qReg=sqTmp, rReg=sXCC, dReg="WorkGroup0", divisor=kernel["StreamKXCCMapping"], tmpSgprRes=sTmpRes, doRemainder=2)) # Check if current XCC requires a remainder WG or not module.add(SCmpLtU32(src0=sgpr(sXCC), src1=sgpr(sGridM), comment="XCCM < Remainder")) module.add(SCSelectB32(dst=sgpr(sGridC), src0=sgpr(sGridC), src1=sgpr(sGridF), comment="Select multiplier")) module.add(SCSelectB32(dst=sgpr(sGridM), src0=0, src1=sgpr(sGridM), comment="Select remainder")) # WG = floor(wg0 / xccm) * xccm + XCCoffset + optional remainder - module.add(scalarStaticDivideAndRemainder(qReg=sgpr("WorkGroup0"), rReg=None, dReg=sgpr("WorkGroup0"), divisor=kernel["StreamKXCCMapping"], tmpSgprRes=sTmpRes, doRemainder=0)) + module.add(scalarStaticDivideAndRemainder(qReg="WorkGroup0", rReg=None, dReg="WorkGroup0", divisor=kernel["StreamKXCCMapping"], tmpSgprRes=sTmpRes, doRemainder=0)) module.add(SMulI32(dst=sgpr(sXCC), src0=sgpr(sXCC), src1=sgpr(sGridC), comment="XCC group id")) module.add(SAddU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=sgpr(sXCC), comment="Add XCC group offset")) module.add(SAddU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=sgpr(sGridM), comment="Add remainder offset")) @@ -92,22 +92,24 @@ class StreamK(Component): """ StreamK code. """ + def __call__(self): + assert(0) @abc.abstractmethod def preLoop(self, writer, kernel): pass @abc.abstractmethod - def graWorkGroup(self, writer, kernel): + def graWorkGroup(self, writer, kernel, tPA, tPB): pass - def skTileIndex(self, writer, kernel, sTmp): + def skTileIndex(self, writer, kernel, sTmp, tPA, tPB): module = Module("StreamK skTileIndex") # Always reset pointers to handle odd-exit case which moves LRO to the upper bank if kernel["PrefetchGlobalRead"]: # not self.prefetchAcrossPersistent - module.add(writer.localReadResetOffsets(kernel, writer.tPA)) - module.add(writer.localReadResetOffsets(kernel, writer.tPB)) + module.add(writer.localReadResetOffsets(kernel, tPA)) + module.add(writer.localReadResetOffsets(kernel, tPB)) module.addComment0("StreamK calculate tile idx and map to WG") @@ -150,6 +152,7 @@ def computeLoadSrd(self, writer, kernel, tc, sTmp): def computeLoadSrdCommon(self, writer, kernel, tc, sTmp): module = Module("StreamK Common computeLoadSrd") + tileStart = sTmp + 2 # StreamK partial tile - offset to tile start index module.add(SMulI32(dst=sgpr(sTmp), src0=sgpr("StreamKLocalStart"), src1="DepthU", comment="StreamK tile start offset")) strideL = writer.strideRef(tc, kernel["ProblemType"]["IndicesSummation"][0]) @@ -223,7 +226,7 @@ def calculateLoopNumIterCommon(self, writer, kernel, loopCounterName, loopIdx, t module = Module("StreamK Common calculateLoopNumIter") # Use StreamK params for loop count - module.add(SSubU32(dst=sgpr(loopCounterName), src0=sgpr("StreamKLocalEnd"), src1=sgpr("StreamKLocalStart"), common="StreamK loop counter = localEnd - localStart")) + module.add(SSubU32(dst=sgpr(loopCounterName), src0=sgpr("StreamKLocalEnd"), src1=sgpr("StreamKLocalStart"), comment="StreamK loop counter = localEnd - localStart")) # Adjust loop count for tail loop if not kernel["NoTailLoop"]: tmpSgpr = tmpSgprInfo.idx @@ -231,29 +234,15 @@ def calculateLoopNumIterCommon(self, writer, kernel, loopCounterName, loopIdx, t loopChar = writer.states.indexChars[kernel["ProblemType"]["IndicesSummation"][unrollIdx]] assert kernel["DepthU"] % 2 == 0 # Assuming DepthU is power of 2, if odd DepthU were supported this divide would need 2 more temp registers for divide - module.add(scalarStaticDivideAndRemainder(qReg=sgpr(tmpSgpr), rReg=sgpr(tmpSgpr+1), dReg=sgpr("SizesSum+%u" % unrollIdx), divisor=kernel["DepthU"], tmpSgprRes=None, doRemainder=2)) - module.add(SCmpEqU32(src0=sgpr(tmpSgpr+1), src1=hex(0), comment="numIter%s == 0"%loopChar )) + module.add(scalarStaticDivideAndRemainder(qReg=tmpSgpr, rReg=tmpSgpr+1, dReg=("SizesSum+%u" % unrollIdx), divisor=kernel["DepthU"], tmpSgprRes=None, doRemainder=2)) + module.add(SCmpEQU32(src0=sgpr(tmpSgpr+1), src1=hex(0), comment="numIter%s == 0"%loopChar )) module.add(SCSelectB32(dst=sgpr(tmpSgpr), src0=0, src1=1, comment="check if size uses tail loop")) - module.add(SCmpEqU32(src0=sgpr("StreamKLocalEnd"), src1=sgpr("ItersPerTile"), comment="Check if WG processes final iteration of tile")) + module.add(SCmpEQU32(src0=sgpr("StreamKLocalEnd"), src1=sgpr("ItersPerTile"), comment="Check if WG processes final iteration of tile")) module.add(SCSelectB32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=0, comment="this WG runs tail loop")) module.add(SSubU32(dst=sgpr(loopCounterName), src0=sgpr(loopCounterName), src1=sgpr(tmpSgpr), comment="Adjust loop counter for tail loop")) return module - @abc.abstractmethod - def recalcLocalWriteAddresses(self, writer, kernel, tc): - pass - - def recalcLocalWriteAddressesCommon(self, writer, kernel, tc): - module = Module("StreamK Common recalcLocalWriteAddresses") - - if getattr(writer, "oriLwa%s" % tc) is None: - setattr(writer, "oriLwa%s" % tc, writer.vgprPool.checkOut(1, "OriLocalWriteddr%s" % tc)) - module.add(VMovB32(dst=vgpr(getattr(writer, "oriLwa%s" % tc)), src=vgpr("LocalWriteAddr%s" % tc), comment="back up LWA for persistent kernel + wider local read")) - - return module - - class StreamKOff(StreamK): kernel = {"StreamK": 0} @@ -261,7 +250,7 @@ def preLoop(self, writer, kernel): module = Module("StreamK Off openLoop") return module - def graWorkGroup(self, writer, kernel): + def graWorkGroup(self, writer, kernel, tPA, tPB): module = Module("StreamK Off graWorkGroup") return module @@ -301,11 +290,6 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module.add(scalarStaticDivideAndRemainder(qReg=quotient, rReg=None, dReg=dividend, divisor=divisor, tmpSgprRes=tmpSgprInfo, doRemainder=0)) return module - - def recalcLocalWriteAddresses(self, writer, kernel, tc): - module = Module("StreamK Off recalcLocalWriteAddresses") - return module - class StreamKBasic(StreamK): kernel = {"StreamK": 1} @@ -327,13 +311,13 @@ def preLoop(self, writer, kernel): return module - def graWorkGroup(self, writer, kernel): + def graWorkGroup(self, writer, kernel, tPA, tPB): module = Module("StreamK Basic graWorkGroup") # StreamK workgroup mapping sTmp = writer.sgprPool.checkOutAligned(4, 2, "SKMappingTemp", preventOverflow=0) - module.add(self.skTileIndex(writer, kernel, sTmp)) + module.add(self.skTileIndex(writer, kernel, sTmp, tPA, tPB)) # Increment StreamK iteration module.add(SMovB32(dst=sgpr("StreamKIter"), src=sgpr(sTmp+2), comment="Increment StreamK Iteration")) @@ -368,12 +352,7 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module = Module("StreamK Basic calculateLoopNumIter") module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) return module - - def recalcLocalWriteAddresses(self, writer, kernel, tc): - module = Module("StreamK Basic recalcLocalWriteAddresses") - module.add(self.recalcLocalWriteAddressesCommon(writer, kernel, tc)) - return module - + class StreamKTwoTileOriginal(StreamK): kernel = {"StreamK": 2} @@ -412,13 +391,13 @@ def preLoop(self, writer, kernel): return module - def graWorkGroup(self, writer, kernel): + def graWorkGroup(self, writer, kernel, tPA, tPB): module = Module("StreamK TwoTileOriginal graWorkGroup") # StreamK workgroup mapping sTmp = writer.sgprPool.checkOutAligned(4, 2, "SKMappingTemp", preventOverflow=0) - module.add(self.skTileIndex(writer, kernel, sTmp)) + module.add(self.skTileIndex(writer, kernel, sTmp, tPA, tPB)) # local end (DP tile) # TODO This line isnt needed? @@ -474,11 +453,6 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) return module - def recalcLocalWriteAddresses(self, writer, kernel, tc): - module = Module("StreamK TwoTileOriginal recalcLocalWriteAddresses") - module.add(self.recalcLocalWriteAddressesCommon(writer, kernel, tc)) - return module - class StreamKTwoTileDPFirst(StreamK): kernel = {"StreamK": 3} @@ -497,7 +471,7 @@ def preLoop(self, writer, kernel): sTmp = writer.sgprPool.checkOut(1, "TotalSKIters", preventOverflow=False) module.add(SMulI32(dst=sgpr(sTmp), src0=sgpr("skTiles"), src1=sgpr("ItersPerTile"), comment="Total SK iters")) module.add(SCmpLtU32(src0=sgpr(sTmp), src1=sgpr("TotalIters"), comment="Check if there are DP tiles to do")) - module.add(SCBranchSCC1(labelName=skInitDone, comment="Done init")) + module.add(SCBranchSCC1(labelName=skInitDone.getLabelName(), comment="Done init")) writer.sgprPool.checkIn(sTmp) # If there are no DP tiles to do, regular SK init @@ -530,13 +504,13 @@ def preLoop(self, writer, kernel): return module - def graWorkGroup(self, writer, kernel): + def graWorkGroup(self, writer, kernel, tPA, tPB): module = Module("StreamK TwoTileDPFirst graWorkGroup") # StreamK workgroup mapping sTmp = writer.sgprPool.checkOutAligned(4, 2, "SKMappingTemp", preventOverflow=0) - module.add(self.skTileIndex(writer, kernel, sTmp)) + module.add(self.skTileIndex(writer, kernel, sTmp, tPA, tPB)) skUpdateDone = Label("SK_UpdateDone", "") # sTmp+3 = Offset to first SK tile @@ -547,11 +521,11 @@ def graWorkGroup(self, writer, kernel): module.add(SAddU32(dst=sgpr(sTmp+1), src0=sgpr(sTmp+1), src1=sgpr("StreamKIter"), comment="Add DP shift")) # if sTmp+1 < sTmp+3, continue DP (add dpShift) module.add(SCmpLtU32(src0=sgpr(sTmp+1), src1=sgpr(sTmp+3), comment="Check if still in DP section")) - module.add(SCBranchSCC1(labelName=skUpdateDone, comment="Done update")) + module.add(SCBranchSCC1(labelName=skUpdateDone.getLabelName(), comment="Done update")) # if StreamKIter >= sTmp+3, continue SK (add skShift?) module.add(SMovB32(dst=sgpr(sTmp+1), src=sgpr(sTmp+2), comment="SK iterations shift")) module.add(SCmpLeU32(src0=sgpr(sTmp+3), src1=sgpr("StreamKIter"), comment="Check if continuing in SK section")) - module.add(SCBranchSCC1(labelName=skUpdateDone, comment="Done update")) + module.add(SCBranchSCC1(labelName=skUpdateDone.getLabelName(), comment="Done update")) # if sTmp+1 > sTmp+3 and StreamKIter < sTmp+3, switch from DP to SK (add dpShift) # iter count after all extra iters have been distributed module.add(SMulI32(dst=sgpr("StreamKIter"), src0=sgpr("StreamKIdx"), src1=sgpr("SKItersPerWG"), comment="StreamK starting iteration (case: after extra iters)")) @@ -613,8 +587,3 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module = Module("StreamK TwoTileDPFirst calculateLoopNumIter") module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) return module - - def recalcLocalWriteAddresses(self, writer, kernel, tc): - module = Module("StreamK TwoTileDPFirst recalcLocalWriteAddresses") - module.add(self.recalcLocalWriteAddressesCommon(writer, kernel, tc)) - return module diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index e6ba449f7b..eddd4ce87e 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -1347,7 +1347,7 @@ def setupNewTile(self, kernel, tensorParametersA, tensorParametersB, isOptNLL=Fa # work-group assignments module.addComment1("global read addresses: work-group") if not forceNoTileCode: - module.add(self.graWorkGroup(kernel)) + module.add(self.graWorkGroup(kernel, tensorParametersA, tensorParametersB)) self.dontAppendCode = forceNoTileCode diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 811bf49e65..e980988f85 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -1906,12 +1906,12 @@ def extractPackedCoord1ToRowStart(self, kernel, packedC1, packedCoordVgpr, store ############################################################################## # Global Read Addresses: WorkGroup ############################################################################## - def graWorkGroup(self, kernel): + def graWorkGroup(self, kernel, tPA, tPB): module = Module("graWorkGroup") module.addComment0("graWorkGroup mapping") skComponent = Component.StreamK.find(self) - module.add(skComponent.graWorkGroup(self, kernel)) + module.add(skComponent.graWorkGroup(self, kernel, tPA, tPB)) gsuComponent = Component.GSU.find(self) module.add(gsuComponent.graWorkGroup(self, kernel)) @@ -6255,8 +6255,8 @@ def recalcLocalWriteAddresses(self, kernel, tP): [tP["localWriteStrideTile"], tP["localWriteStrideUnroll"]] ) tP["localWriteInstruction"] = self.memoryInstructions["LocalWrite"][newInstIdx] - skComponent = Component.StreamK.find(self) - module.add(skComponent.recalcLocalWriteAddresses(self, kernel)) + loopComponent = Component.PersistentLoop.find(self) + module.add(loopComponent.recalcLocalWriteAddresses(self, kernel, tc)) # local write tile assignments module.add(self.lwaTileAssignment(kernel, tP)) @@ -6290,6 +6290,9 @@ def recalcLocalReadAddressesAB(self, kernel, tPA, tPB): # this decrease performance since it require more loop to handle continuous k in each thread. # reCalculating localread address because we disable wider local read in tail loop if ((self.states.numReadsIterCoalescedA > 1 or self.states.numReadsIterCoalescedB > 1)): + loopComponent = Component.PersistentLoop.find(self) + imod.add(loopComponent.recalcLocalReadAddressesAB(self, kernel)) + self.states.numReadsIterCoalescedA = 1 self.states.numReadsIterCoalescedB = 1 if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index f4260b6f0e..0013e3449c 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -1940,6 +1940,8 @@ def assignDerivedParameters(state): state["SynchronizerSizeCheck"] = 1 # state["BatchSizeEqual"] = 1 + isa = tuple(state["ISA"]) + if state["StreamK"] != 0: if state["MIWaveGroup"][0] * state["MIWaveGroup"][1] != 4: reject(state, "Stream-K requries MIWaveGroup0*MIWaveGroup1=4") @@ -1989,8 +1991,6 @@ def assignDerivedParameters(state): state["1LDSBuffer"] = 1 print2("\nSet SIA=2, force PrefetchLocalRead=1, ExpandPointerSwap=1, 1LDSBuffer=1") - isa = tuple(state["ISA"]) - if state["WavefrontSize"] == 32 and not globalParameters["ArchCaps"][isa]["HasWave32"]: reject(state, "WavefrontSize=32 not supported for ISA {}".format(isa)) From 0d5cec765e9abb8484fced5eb566e957d3cc3f82 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Fri, 28 Jun 2024 17:31:43 +0000 Subject: [PATCH 11/36] Store branches and close persistent loop --- .../Tensile/Components/PersistentLoop.py | 19 ++- tensilelite/Tensile/Components/StreamK.py | 135 +++++++++++++++++- tensilelite/Tensile/KernelWriter.py | 15 +- tensilelite/Tensile/KernelWriterAssembly.py | 13 +- 4 files changed, 170 insertions(+), 12 deletions(-) diff --git a/tensilelite/Tensile/Components/PersistentLoop.py b/tensilelite/Tensile/Components/PersistentLoop.py index cf6a474479..bfd42bab44 100644 --- a/tensilelite/Tensile/Components/PersistentLoop.py +++ b/tensilelite/Tensile/Components/PersistentLoop.py @@ -20,7 +20,7 @@ # CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ################################################################################ -from ..TensileInstructions import Module, Label, VMovB32, vgpr +from ..TensileInstructions import Module, Label, VMovB32, vgpr, sgpr, SCmpGeU32 from ..Component import Component import abc @@ -43,6 +43,10 @@ def recalcLocalWriteAddresses(self, writer, kernel, tc): def recalcLocalReadAddressesAB(self, writer, kernel): pass + @abc.abstractmethod + def closePersistentLoop(self, writer, kernel): + pass + class PersistentLoopOff(PersistentLoop): kernel = {"StreamK": 0} @@ -59,8 +63,13 @@ def recalcLocalReadAddressesAB(self, writer, kernel): module = Module("PersistentLoop Off recalcLocalReadAddressesAB") return module + def closePersistentLoop(self, writer, kernel): + module = Module("PersistentLoop Off closePersistentLoop") + return module + class PersistentLoopOn(PersistentLoop): + # Stream-K persistent loop @classmethod def matches(cls, writer, debug=False): @@ -106,3 +115,11 @@ def recalcLocalReadAddressesAB(self, writer, kernel): module.add(VMovB32(dst=vgpr(writer.oriLraB), src=vgpr("LocalReadAddrB"), comment="back up LRA for persistent kernel + wider local read")) return module + + def closePersistentLoop(self, writer, kernel): + module = Module("PersistentLoop On closePersistentLoop") + # endIter = "StreamKIterEnd" if kernel["StreamK"] == 1 else "TotalIters" + endIter = "TotalIters" if kernel["StreamK"] == 2 else "StreamKIterEnd" + module.add(SCmpGeU32(src0=sgpr("StreamKIter"), src1=sgpr(endIter), comment="Check if done all StreamK iterations")) + module.add(writer.longBranchScc0(Label("PersistentLoopStart", ""), posNeg=-1)) + return module diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index f5b9cf55e6..46b2bf75dd 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -22,7 +22,9 @@ from ..TensileInstructions import Module, Label, SAddU32, RegisterPoolResource, sgpr, scalarStaticDivideAndRemainder, \ SCmpLtU32, SCSelectB32, sMagicDivAlg2, SMulI32, SSubU32, SMinU32, SMovB32, SCBranchSCC1, SCmpLeU32, VMovB32, vgpr, \ - SAddCU32, SCmpGtU32, SCMovB32, SAddI32, SCmpEQU32 + SAddCU32, SCmpGtU32, SCMovB32, SAddI32, SCmpEQU32, SCBranchSCC0, SLShiftLeftB32, SLoadB32, SWaitCnt, SMEMModifiers, \ + log2 +# from ..TensileInstructions.Containers import SMEMModifiers from ..Component import Component import abc @@ -243,6 +245,98 @@ def calculateLoopNumIterCommon(self, writer, kernel, loopCounterName, loopIdx, t return module + @abc.abstractmethod + def storeBranches(self, writer, kernel, skPartialsLabel): + pass + + def storeBranchesCommon(self, writer, kernel, skPartialsLabel): + module = Module("StreamK Common storeBranches") + + # No branches for atomic mode + if kernel["StreamKAtomic"]: + return module + + skFixupLabel = Label(label=writer.labels.getNameInc("SK_Fixup"), comment="") + skStoreLabel = Label(label=writer.labels.getNameInc("SK_Store"), comment="") + + # StreamK store branches + tmpSgpr = self.sgprPool.checkOut(4, "globalWriteElements", preventOverflow=0) + # if we did not start the tile, store partials + # branch to beta == 0 store path + module.add(SCmpEQU32(src0=sgpr("StreamKLocalStart"), src1=0, comment="does wg start tile?")) + module.add(SCBranchSCC0(labelName=skPartialsLabel.getLabelName(), comment="Branch if not start tile, store partials")) + + if kernel["DebugStreamK"] & 1 == 0: + # if we started and finished the tile, regular store code + # branch to regular store code, skip fixup step + module.add(SCmpEQU32(src0=sgpr("StreamKLocalEnd"), src1=sgpr("ItersPerTile"), comment="does wg finish tile?")) + module.add(SCBranchSCC1(labelName=skStoreLabel.getLabelName(), comment="Branch if started and finished tile, go to regular store code")) + + # if we started the tile but did not finish it, fix up step + # run fixup code before regular store code + sCtaIdx = writer.sgprPool.checkOut(1, "CtaIdx", preventOverflow=0) # self.defineSgpr("CtaIdx", 1) + module.add(SAddU32(dst=sgpr(sCtaIdx), src0=sgpr("StreamKIdx"), src1=1, comment="input partial tile index")) + + sFixupEnd = writer.sgprPool.checkOut(1, "FixupEnd", preventOverflow=0) # self.defineSgpr("CtaEnd", 1) + module.add(sMagicDivAlg2(tmpSgpr, sgpr("StreamKIterEnd"), sgpr("MagicNumberItersPerTile"), sgpr("MagicShiftItersPerTile"))) + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=sgpr(tmpSgpr), src1=sgpr("ItersPerTile"), comment="start iteration of partial tile")) + module.add(SSubU32(dst=sgpr(sFixupEnd), src0=sgpr("StreamKIterEnd"), src1=sgpr(tmpSgpr), comment="calc iterations completed by this WG")) + + module.add(skFixupLabel) + + # Check flag + module.add(SLShiftLeftB32(dst=sgpr(tmpSgpr), src=sgpr(sCtaIdx), shiftHex=log2(4), comment="flag offset based on CTA index")) + module.add(SLoadB32(dst=sgpr(tmpSgpr+2), base=sgpr("AddressFlags", 2), soffset=sgpr(tmpSgpr), smem=SMEMModifiers(glc=1), comment="get flag")) + + module.add(SWaitCnt(lgkmcnt=0, comment="wait for flag load")) + if kernel["DebugStreamK"] & 2 == 0: + module.add(SCmpEQU32(src0=sgpr(tmpSgpr+2), src1=1, comment="check if ready")) + module.add(SCBranchSCC0(labelName=skFixupLabel.getLabelName(), comment="if flag not set, wait and check again")) + + writer.sgprPool.checkIn(tmpSgpr) + + # TODO FIXUP STEP!!!!!!!!!!!!!!! + fixupEdge = [False] # Temporary hack to test no edge variant + kStr += self.fixupStep(kernel, vectorWidths, elements, fixupEdge, tmpVgpr, tmpCVTVgpr, sCtaIdx, skStoreLabel) + + if kernel["StreamK"] >= 2: + sIterCount = writer.sgprPool.checkOut(1, "iterCount", preventOverflow=0) + module.add(SAddU32(dst=sgpr(sIterCount), src0=sgpr("SKItersPerWG"), src1=1, comment="Add extra iter")) + module.add(SCmpLtU32(src0=sgpr(sCtaIdx), src1=sgpr("skExtraIters"), comment="Check if next WG had an extra iteration")) + module.add(SCSelectB32(dst=sgpr(sIterCount), src0=sgpr(sIterCount), src1=sgpr("SKItersPerWG"), comment="Select correct number of iterations for next WG")) + module.add(SAddU32(dst=sgpr(sFixupEnd), src0=sgpr(sFixupEnd), src1=sgpr(sIterCount), comment="next partial tile iteration")) + writer.sgprPool.checkIn(sIterCount) + module.add(SAddU32(dst=sgpr(sCtaIdx), src0=sgpr(sCtaIdx), src1=1, comment="next partial tile index")) + if kernel["StreamK"] == 1: + module.add(SAddU32(dst=sgpr(sFixupEnd), src0=sgpr(sFixupEnd), src1=sgpr("SKItersPerWG"), comment="next partial tile iteration")) + module.add(SCmpLtU32(src0=sgpr(sFixupEnd), src1=sgpr("ItersPerTile"), comment="done loading partial tiles?")) + module.add(SCBranchSCC1(labelName=skFixupLabel.getLabelName(), comment="Branch to continue fixup loop")) + + writer.sgprPool.checkIn(sFixupEnd) + writer.sgprPool.checkIn(sCtaIdx) + + module.add(skStoreLabel) + + return module + + @abc.abstractmethod + def writePartials(self, writer, kernel, skPartialsLabel): + pass + + def writePartialsCommon(self, writer, kernel, skPartialsLabel): + module = Module("StreamK Common writePartials") + + # No partials for atomic mode + if kernel["StreamKAtomic"]: + return module + + module.add(skPartialsLabel.getLabelName()) + if kernel["DebugStreamK"] & 2 == 0: + fixupEdge = [False] # Temporary hack to test no edge variant + kStr += self.writePartials(kernel, vectorWidths, elements, fixupEdge, atomic, tmpVgpr, tmpCVTVgpr, isOptNLL, endLabel) + + return module + class StreamKOff(StreamK): kernel = {"StreamK": 0} @@ -290,6 +384,14 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module.add(scalarStaticDivideAndRemainder(qReg=quotient, rReg=None, dReg=dividend, divisor=divisor, tmpSgprRes=tmpSgprInfo, doRemainder=0)) return module + + def storeBranches(self, writer, kernel, skPartialsLabel): + module = Module("StreamK Off storeBranches") + return module + + def writePartials(self, writer, kernel, skPartialsLabel): + module = Module("StreamK Off writePartials") + return module class StreamKBasic(StreamK): kernel = {"StreamK": 1} @@ -352,7 +454,17 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module = Module("StreamK Basic calculateLoopNumIter") module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) return module + + def storeBranches(self, writer, kernel, skPartialsLabel): + module = Module("StreamK Basic storeBranches") + module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel)) + return module + def writePartials(self, writer, kernel, skPartialsLabel): + module = Module("StreamK Basic writePartials") + module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel)) + return module + class StreamKTwoTileOriginal(StreamK): kernel = {"StreamK": 2} @@ -452,6 +564,17 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module = Module("StreamK TwoTileOriginal calculateLoopNumIter") module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) return module + + def storeBranches(self, writer, kernel, skPartialsLabel): + module = Module("StreamK TwoTileOriginal storeBranches") + module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel)) + return module + + def writePartials(self, writer, kernel, skPartialsLabel): + module = Module("StreamK TwoTileOriginal writePartials") + module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel)) + return module + class StreamKTwoTileDPFirst(StreamK): kernel = {"StreamK": 3} @@ -587,3 +710,13 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module = Module("StreamK TwoTileDPFirst calculateLoopNumIter") module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) return module + + def storeBranches(self, writer, kernel, skPartialsLabel): + module = Module("StreamK TwoTileDPFirst storeBranches") + module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel)) + return module + + def writePartials(self, writer, kernel, skPartialsLabel): + module = Module("StreamK TwoTileDPFirst writePartials") + module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel)) + return module diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index eddd4ce87e..e523aee654 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -1517,7 +1517,7 @@ def setupNewTile(self, kernel, tensorParametersA, tensorParametersB, isOptNLL=Fa module.add(self.localReadInitPointers(kernel, tensorParametersA, tensorParametersB)) if self.do["executeToInitEnd"]: - module.add(self.functionEnd(False)) + module.add(self.functionEnd(kernel, addLabel=False)) #################################### # prefetch: unrolled loop prefix @@ -1749,7 +1749,7 @@ def noLoadLoop( self, kernel, tensorParametersA, tensorParametersB, isOptNLL, is module.add(self.noLoadLoopBody(kernel, tensorParametersA, tensorParametersB, pack, isOptNLL, isNGLL, NLLfirst, NLLlast)) if self.do["executeToLoopEnd"] and isOptNLL: - module.add(self.functionEnd(False)) + module.add(self.functionEnd(kernel, addLabel=False)) # Close code is necessary for both first and last (NGLL case(=NLLfirst) needs label) module.add(self.closeSumAtLeastUnroll(kernel, tensorParametersA, tensorParametersB, prefetch=False, isOptNLL=isOptNLL, isNGLL=isNGLL)) @@ -2083,7 +2083,7 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): module.add(self.setupNewTile(kernel, tensorParametersA, tensorParametersB, isOptNLL=False)) if self.do["executeToPrefetchEnd"]: - module.add(self.functionEnd(False)) + module.add(self.functionEnd(kernel, addLabel=False)) pack = [ Module() for i in range (self.states.numVgprBuffer) ] self.preLoopLocalWriteCode = None @@ -2103,7 +2103,7 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): # we need an extra barrier to ensure that the ds_reads (either for SR or MFMA) from previous iteration # have finished before we generate the prefetch for the next summation index. if kernel["StreamK"] > 0 or self.states.actualSummationLoops>1: - module.add(SBarrier()) + module.add(SBarrier(comment="For stream-k / persistent loop")) # local write self.preLoopLocalWriteCode = self.preLoopLocalWriteDo(kernel, tensorParametersA, tensorParametersB) @@ -2385,7 +2385,7 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): (self.states.lastValuAB, self.states.lastVgprForReads)) if self.do["executeToLoopEnd"]: - module.add(self.functionEnd(False)) + module.add(self.functionEnd(kernel, addLabel=False)) # extra summation loops: global increment and close for i in reversed(range(self.states.otherSummationLoops)): @@ -2443,7 +2443,6 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): module.addComment1("LocalSplitU: global write") module.add(self.localSplitUGlobalWrite(kernel, tensorParametersA, tensorParametersB)) - else: #################################### # NOT LocalSplitU @@ -2458,7 +2457,7 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): module.add(self.notLocalSplitUGlobalWrite(kernel, tensorParametersA, tensorParametersB)) # function suffix - module.add(self.functionEnd(True)) + module.add(self.functionEnd(kernel, addLabel=True)) module.add(self.functionSuffix(kernel)) # Tensile pass @@ -4372,7 +4371,7 @@ def closePrefetchGlobalRead2(self): # Function End ############################################################################## @abc.abstractmethod - def functionEnd(self, addLabel=True): + def functionEnd(self, kernel, addLabel=True): return "" ############################################################################## diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index e980988f85..ad99a75597 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -5053,7 +5053,7 @@ def closeSumAtLeastUnroll(self, kernel, tPA, tPB, prefetch, isOptNLL, isNGLL): self.cleanupGlobalWrite(kernel) module.addSpaceLine() - module.add(self.functionEnd(False)) + module.add(self.functionEnd(kernel, addLabel=False)) module.add(Label("OptNLL_End", "")) else: @@ -8322,6 +8322,11 @@ def globalWriteElements(self, kernel, tPA, tPB, vectorWidths_2, vectorWidths_1, biasDims=None): if not self.do["PostLoop"]: return Module("GlobalWriteElements (Empty)") module = Module("GlobalWriteElements") + + skPartialsLabel = Label(label=self.labels.getNameInc("SK_Partials"), comment="") + skComponent = Component.StreamK.find(self) + module.add(skComponent.storeBrances(self, kernel, skPartialsLabel)) + module.addComment2("Global Write Elements") if self.states.numStoreSgprToLoad or self.states.numStoreSgprToLoad2: # Wait for kernel args module.add(SWaitCnt(lgkmcnt=0, comment="wait for %u bytes of kern args."%((self.states.numStoreSgprToLoad+self.states.numStoreSgprToLoad2) * 4))) @@ -8883,6 +8888,8 @@ def findInstCount(module, targetItem, count): self.sgprPool.checkIn(activationSetPCStruct.sgprOffsetActivation) self.sgprPool.checkIn(activationSetPCStruct.sgprOffsetBack) + module.add(skComponent.writePartials(self, kernel, skPartialsLabel)) + # End label module.add(endLabel) if self.states.BiasDim == 3: @@ -10195,8 +10202,10 @@ def getActivationActivationComputeType(self, kernel, activation, activationTypeS ############################################################################## # Function End ############################################################################## - def functionEnd(self, addLabel=True): + def functionEnd(self, kernel, addLabel=True): imod = Module() + loopComponent = Component.PersistentLoop.find(self) + imod.add(loopComponent.closePersistentLoop(self, kernel)) if addLabel: imod.add(Label("KernelEnd", "")) imod.add(SEndpgm(comment="Kernel End")) From 229674809953f3f7ae247279a3762a76b915b750 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Tue, 2 Jul 2024 20:31:36 +0000 Subject: [PATCH 12/36] Add code for workspace SRD, working on partials write, some updates/fixes --- tensilelite/Tensile/Components/StreamK.py | 86 ++++++++++++++++----- tensilelite/Tensile/KernelWriterAssembly.py | 11 +-- 2 files changed, 69 insertions(+), 28 deletions(-) diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index 46b2bf75dd..80b058c8f6 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -246,10 +246,10 @@ def calculateLoopNumIterCommon(self, writer, kernel, loopCounterName, loopIdx, t return module @abc.abstractmethod - def storeBranches(self, writer, kernel, skPartialsLabel): + def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements): pass - def storeBranchesCommon(self, writer, kernel, skPartialsLabel): + def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, elements): module = Module("StreamK Common storeBranches") # No branches for atomic mode @@ -320,10 +320,10 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel): return module @abc.abstractmethod - def writePartials(self, writer, kernel, skPartialsLabel): + def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements): pass - def writePartialsCommon(self, writer, kernel, skPartialsLabel): + def writePartialsCommon(self, writer, kernel, skPartialsLabel, vectorWidths, elements): module = Module("StreamK Common writePartials") # No partials for atomic mode @@ -332,9 +332,53 @@ def writePartialsCommon(self, writer, kernel, skPartialsLabel): module.add(skPartialsLabel.getLabelName()) if kernel["DebugStreamK"] & 2 == 0: - fixupEdge = [False] # Temporary hack to test no edge variant - kStr += self.writePartials(kernel, vectorWidths, elements, fixupEdge, atomic, tmpVgpr, tmpCVTVgpr, isOptNLL, endLabel) - + return module + + # fixupEdge = [False] # Temporary hack to test no edge variant + edges = [False] + + partialsLabels = {} + for edge in edges: + partialsLabels[edge] = Label(writer.labels.getNameInc("GW_Partials_E%u" % ( 1 if edge else 0), "")) + + if False in edges and True in edges: + with self.allocTmpSgpr(4) as tmpSgprInfo: + module.add(writer.checkIsEdge(kernel, tmpSgprInfo, partialsLabels[True], partialsLabels[True])) + + for edge in edges: + module.add(partialsLabels[edge]) + kStr += self.computeWorkspaceSrd(writer, kernel, sgpr("StreamKIdx")) + kStr += self.partialsWriteProcedure(kernel, vectorWidths, elements, False, False, edge, atomic, tmpVgpr, tmpCVTVgpr, isOptNLL, endLabel) + + return module + + + ############################################################################## + # Workspace SRD for FixupStep + ############################################################################## + def computeWorkspaceSrd(self, writer, kernel, sCtaIdx, tmpSgpr = None): + module = Module("StreamK Common computeWorkspaceSrd") + + # Base Address + module.add(SMovB32(dst=sgpr("SrdWS+0"), src=sgpr("AddressWS+0"), comment="init SRD base address (lower)")) + module.add(SMovB32(dst=sgpr("SrdWS+1"), src=sgpr("AddressWS+1"), comment="init SRD base address (upper) + other fields")) + module.add(SMovB32(dst=sgpr("SrdWS+2"), src="BufferOOB", comment="")) + module.add(SMovB32(dst=sgpr("SrdWS+3"), src="Srd127_96", comment="Set bits 127_96 in post-loop SRD")) + + tmpLocal = None + if tmpSgpr == None: + tmpLocal = writer.sgprPool.checkOut(1, "SKMappingTemp", preventOverflow=0) + tmpSgpr = tmpLocal + + assert kernel["BufferStore"] + module.addSpaceLine() + module.add(SMulI32(dst=sgpr(tmpSgpr), src0=hex(kernel["MacroTile0"]*kernel["MacroTile1"]*writer.states.bpeCinternal), src1=sCtaIdx, comment="Offset to correct partials tile")) + module.add(SAddU32(dst=sgpr("SrdWS+0"), src0=sgpr("SrdWS+0"), src1=sgpr(tmpSgpr), comment="add lo to SRD")) + module.add(SAddCU32(dst=sgpr("SrdWS+1"), src0=sgpr("SrdWS+1"), src1=0, comment="add hi to SRD")) + + if tmpLocal is not None: + writer.sgprPool.checkIn(tmpLocal) + return module class StreamKOff(StreamK): @@ -385,11 +429,11 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr return module - def storeBranches(self, writer, kernel, skPartialsLabel): + def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements): module = Module("StreamK Off storeBranches") return module - def writePartials(self, writer, kernel, skPartialsLabel): + def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements): module = Module("StreamK Off writePartials") return module @@ -455,14 +499,14 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) return module - def storeBranches(self, writer, kernel, skPartialsLabel): + def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements): module = Module("StreamK Basic storeBranches") - module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel)) + module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) return module - def writePartials(self, writer, kernel, skPartialsLabel): + def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements): module = Module("StreamK Basic writePartials") - module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel)) + module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) return module class StreamKTwoTileOriginal(StreamK): @@ -565,14 +609,14 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) return module - def storeBranches(self, writer, kernel, skPartialsLabel): + def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements): module = Module("StreamK TwoTileOriginal storeBranches") - module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel)) + module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) return module - def writePartials(self, writer, kernel, skPartialsLabel): + def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements): module = Module("StreamK TwoTileOriginal writePartials") - module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel)) + module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) return module @@ -711,12 +755,12 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) return module - def storeBranches(self, writer, kernel, skPartialsLabel): + def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements): module = Module("StreamK TwoTileDPFirst storeBranches") - module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel)) + module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) return module - def writePartials(self, writer, kernel, skPartialsLabel): + def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements): module = Module("StreamK TwoTileDPFirst writePartials") - module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel)) + module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) return module diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index ad99a75597..0f80368a5c 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -4641,7 +4641,6 @@ def mfmaIter(self, kernel, tPA, tPB, u, innerUnroll, unrollLoopIdx = 0, unrollId iui = 0 abReg = self.vgprPool.checkOutAligned(2, 2, "abReg") - #sgprShift = self.getTmpSgpr(3).idx() with self.allocTmpSgpr(3) as tmpSgprInfo: sgprShift = tmpSgprInfo.idx sgpr64bIdx = sgprShift + 1 @@ -8102,8 +8101,6 @@ def notLocalSplitUGlobalWrite(self, kernel, tPA, tPB): vectorWidths = [fullVw, edgeVw] vectorWidths_1 = [fullVw, edgeVw_1] - - noGSUBranch = (kernel["GlobalSplitU"] == 0) module = Module("notLocalSplitUGlobalWrite") module.add(self.globalWriteElements(kernel, tPA, tPB, vectorWidths, vectorWidths_1, elements, elements_1, noGSUBranch=noGSUBranch)) @@ -8185,7 +8182,7 @@ def checkIsBetaZero(self, kernel, tmpSgprInfo, betaLabel, isLongBranch=False, pl ############################################################################## # checkIsEdge - # tmpSgpr must have at least 6 free SGPR + # tmpSgpr must have at least 4 free SGPR # isEdgeTarget is the branch target if edges are required ############################################################################## def checkIsEdge(self, kernel, tmpSgprInfo, isEdgeTargetMT0, isEdgeTargetMT1, isLongBranch=False, placeHolder=None): @@ -8325,7 +8322,7 @@ def globalWriteElements(self, kernel, tPA, tPB, vectorWidths_2, vectorWidths_1, skPartialsLabel = Label(label=self.labels.getNameInc("SK_Partials"), comment="") skComponent = Component.StreamK.find(self) - module.add(skComponent.storeBrances(self, kernel, skPartialsLabel)) + module.add(skComponent.storeBranches(self, kernel, skPartialsLabel, vectorWidths_1, elements_1)) module.addComment2("Global Write Elements") if self.states.numStoreSgprToLoad or self.states.numStoreSgprToLoad2: # Wait for kernel args @@ -8735,7 +8732,7 @@ def globalWriteElements(self, kernel, tPA, tPB, vectorWidths_2, vectorWidths_1, if kernel["ProblemType"]["DestDataType"].isBFloat16() and kernel["ProblemType"]["HighPrecisionAccumulate"]: cvtVgpr = self.vgprPool.checkOut(4) cvtVgprStruct = self.BF16CVTVgprStruct(vgprBf16Temp=cvtVgpr, vgprBf16Mask=(cvtVgpr+1), \ - vgprFp32Nan=(cvtVgpr+2), vgprBf16Inc=(cvtVgpr+3)) + vgprFp32Nan=(cvtVgpr+2), vgprBf16Inc=(cvtVgpr+3)) elif kernel["ProblemType"]["DestDataType"].isFloat8() and kernel["ProblemType"]["HighPrecisionAccumulate"]: cvtVgpr = self.vgprPool.checkOut(4) cvtVgprStruct = self.FP8CVTVgprStruct(vgprFp8Temp=cvtVgpr, vgprFp8NanInf=(cvtVgpr+1), \ @@ -8888,7 +8885,7 @@ def findInstCount(module, targetItem, count): self.sgprPool.checkIn(activationSetPCStruct.sgprOffsetActivation) self.sgprPool.checkIn(activationSetPCStruct.sgprOffsetBack) - module.add(skComponent.writePartials(self, kernel, skPartialsLabel)) + module.add(skComponent.writePartials(self, kernel, skPartialsLabel, vectorWidths_1, elements_1)) # End label module.add(endLabel) From ac84fa99098b3d7d8910f11f7107a5b22c563706 Mon Sep 17 00:00:00 2001 From: adityalj <167943392+adityalj@users.noreply.github.com> Date: Mon, 15 Jul 2024 17:33:18 -0400 Subject: [PATCH 13/36] =?UTF-8?q?Support=20for=20predicate=20arithmetic=20?= =?UTF-8?q?intensity=20added=20on=20top=20of=20streamK=20br=E2=80=A6=20(#3?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Support for predicate arithmetic intensity added on top of streamK branch in Alex's repo --- tensilelite/Tensile/Common.py | 9 +++ tensilelite/Tensile/Contractions.py | 6 ++ .../include/Tensile/ContractionProblem.hpp | 8 +- .../Tensile/ContractionProblemPredicates.hpp | 77 +++++++++++++++++++ .../Serialization/ContractionPredicates.hpp | 14 ++++ .../Source/lib/source/ContractionProblem.cpp | 37 +++++++++ 6 files changed, 150 insertions(+), 1 deletion(-) diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index 88568fb832..a74276b2eb 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -700,6 +700,12 @@ def getArchitectureName(gfxName): # 1 indicates no assertion (since all sizes are multiples of 1) "AssertFree1ElementMultiple" : [1,2,4,8,16], + # Assertions that require arithmetic intensity to be specified value. + # Arithmetic intensity measures the ratio of computation to memory bandwidth required for a problem. + # These predicates can be used to adjust solution selection compute-bound or memory-bound problems. + "AssertAIGreaterThanEqual": -1, + "AssertAILessThanEqual": -1, + # Stagger the start summation position of the tiles. # Elements from the summation dimension are loaded at offsets rather than all starting at 0. # StaggerU is the max 'clicks' of StaggerUStride bytes where each wg starts ; see StaggerUMapping @@ -1130,6 +1136,9 @@ def getArchitectureName(gfxName): {"AssertSummationElementMultiple": [ 1 ] }, {"AssertFree0ElementMultiple": [ 1 ] }, {"AssertFree1ElementMultiple": [ 1 ] }, + + {"AssertAIGreaterThanEqual": [-1]}, + {"AssertAILessThanEqual": [-1]}, {"StaggerU": [ 32 ] }, # recommend [0,32] {"StaggerUStride": [ 256 ] }, # recommend 256 for V10,V20 diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index e20320b918..48b679277c 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -384,6 +384,12 @@ class ProblemPredicate(Properties.Predicate): @classmethod def FromOriginalKeyPair(cls, pair): (key, value) = pair + # Arithmetic intensity assertions + if key == "AssertAIGreaterThanEqual": + return cls("AIGreaterThanEqual", value=value) if value > 0 else None + if key == "AssertAILessThanEqual": + return cls("AILessThanEqual", value=value) if value > 0 else None + if key.endswith('Multiple'): if value == 1: return None diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem.hpp index 494227f05e..8496379dba 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblem.hpp @@ -1027,6 +1027,11 @@ namespace Tensile return m_eligibleForPK; } + double arithmeticIntensity() const + { + return m_arithmeticIntensity; + } + virtual std::vector const constants() const { std::vector c = {{"alpha", m_alphaType}, {"beta", m_betaType}}; @@ -1103,7 +1108,7 @@ namespace Tensile KernelLanguage m_kernelLanguage = KernelLanguage::Any; PerformanceMetric m_performanceMetric = PerformanceMetric::DeviceEfficiency; - + double m_arithmeticIntensity; DataType m_alphaType = DataType::None; // if not assigned, will follow d-type DataType m_betaType = DataType::None; // for bwd-compatible DataType m_scaleAType = DataType::None; // if not assigned, will follow alpha-type @@ -1145,6 +1150,7 @@ namespace Tensile void normalize(); void normalizeSparse(); + void calcArithmeticIntensity(); void consistencyCheck() const; diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblemPredicates.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblemPredicates.hpp index ab7b63b939..7a52365097 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblemPredicates.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionProblemPredicates.hpp @@ -1015,6 +1015,83 @@ namespace Tensile } }; + struct AIGreaterThanEqual + : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + + double value; + + AIGreaterThanEqual() = default; + AIGreaterThanEqual(double value) + : value(value) + { + } + + static std::string Type() + { + return "AIGreaterThanEqual"; + } + + virtual bool operator()(ContractionProblemGemm const& problem) const override + { + return problem.arithmeticIntensity() >= value; + } + + virtual bool debugEval(ContractionProblemGemm const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.arithmeticIntensity() << " >= " << value + << ") == " << rv; + + return rv; + } + }; + + struct AILessThanEqual : public Predicate_CRTP + { + enum + { + HasIndex = false, + HasValue = true + }; + + double value; + + AILessThanEqual() = default; + AILessThanEqual(double value) + : value(value) + { + } + + static std::string Type() + { + return "AILessThanEqual"; + } + + virtual bool operator()(ContractionProblemGemm const& problem) const override + { + return problem.arithmeticIntensity() <= value; + } + + virtual bool debugEval(ContractionProblemGemm const& problem, + std::ostream& stream) const override + { + bool rv = (*this)(problem); + + stream << *this << ": (" << problem.arithmeticIntensity() << " <= " << value + << ") == " << rv; + + return rv; + } + }; + struct AlphaValue : public Predicate_CRTP { enum diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionPredicates.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionPredicates.hpp index 0bfdb91287..1e811bef3d 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionPredicates.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Serialization/ContractionPredicates.hpp @@ -75,6 +75,8 @@ namespace Tensile Base::template Pair(), Base::template Pair(), Base::template Pair(), + Base::template Pair(), + Base::template Pair(), Base::template Pair(), Base::template Pair(), Base::template Pair(), @@ -245,6 +247,18 @@ namespace Tensile { }; + template + struct MappingTraits + : public AutoMappingTraits + { + }; + + template + struct MappingTraits + : public AutoMappingTraits + { + }; + template struct MappingTraits : public AutoMappingTraits diff --git a/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp b/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp index 03e65486a1..9a67773bfd 100644 --- a/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp +++ b/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp @@ -608,6 +608,7 @@ namespace Tensile m_beta); // Set enum using beta to potentially allow for faster solutions consistencyCheck(); normalize(); + calcArithmeticIntensity(); } size_t ContractionProblemGemm::toAPos(size_t idx) const @@ -1065,6 +1066,42 @@ namespace Tensile TENSILE_ASSERT_EXC(dUse == 1); } + void ContractionProblemGemm::calcArithmeticIntensity() + { + size_t problemSize = 1; + for(size_t i = 0; i < m_problemSizes.size(); ++i) + { + problemSize *= m_problemSizes[i]; + } + double gflop = 2 * problemSize * 1e-9; + + size_t aSize = 1; + for(size_t i = 0; i < m_a.dimensions(); ++i) + { + aSize *= m_a.sizes()[i]; + } + size_t bSize = 1; + for(size_t i = 0; i < m_b.dimensions(); ++i) + { + bSize *= m_b.sizes()[i]; + } + size_t cSize = 1; + for(size_t i = 0; i < m_c.dimensions(); ++i) + { + cSize *= m_c.sizes()[i]; + } + if(m_beta != 0) // If problem includes beta, update gflops and gbytes + { + gflop += 2 * cSize * 1e-9; // Include (+ beta * C) in gflops + cSize *= 2; // Include read C and write D in gbytes + } + double gbyte + = (aSize * m_a.elementBytes() + bSize * m_b.elementBytes() + cSize * m_c.elementBytes()) + * 1e-9; + + m_arithmeticIntensity = gflop / gbyte; + } + size_t ContractionProblemGemm::freeSizeA(size_t idx) const { return m_freeSizesA.at(idx); From 259f6b4ecc19dd636f614def601ff5c4e7628d7e Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Tue, 16 Jul 2024 20:13:22 +0000 Subject: [PATCH 14/36] Write partials and some bug fixes --- tensilelite/Tensile/AsmStoreState.py | 36 +- .../Components/NotLocalFullTileElements.py | 50 +- tensilelite/Tensile/Components/StreamK.py | 485 +++++++++++++++++- tensilelite/Tensile/KernelWriter.py | 10 + tensilelite/Tensile/KernelWriterAssembly.py | 79 ++- .../TensileInstructions/RegisterPool.py | 7 + 6 files changed, 568 insertions(+), 99 deletions(-) diff --git a/tensilelite/Tensile/AsmStoreState.py b/tensilelite/Tensile/AsmStoreState.py index 75c654fb62..60f302c8a3 100644 --- a/tensilelite/Tensile/AsmStoreState.py +++ b/tensilelite/Tensile/AsmStoreState.py @@ -39,7 +39,7 @@ class StoreState: # the generation of the store code. ############################################################################## class StoreConstConfig: - def __init__(self, kernelWriter, kernel, ss, gwvw, edge, beta, atomic): + def __init__(self, kernelWriter, kernel, ss, gwvw, edge, beta, atomic, isWorkspace=False): self.gwvw = gwvw self.lsu = kernel["LocalSplitU"] @@ -76,14 +76,18 @@ def __init__(self, kernelWriter, kernel, ss, gwvw, edge, beta, atomic): self.numElementsPerBatchLimitedBySgprs = 1 # dummy value #assert self.numElementsPerBatchLimitedBySgprs > 0, "numElementsPerBatchLimitedBySgprs=0 for %s"%self.kernelName + bpeC = kernelWriter.states.bpeCexternal + if isWorkspace: + bpeC = kernelWriter.states.bpeCinternal + if atomic: # flat atomics have another VGPR to allow different data for return# regsPerElement = 2 if kernel["BufferStore"] else (3 + 1) # + 1 for alignment # The atomic loop processes multiple elements in single instruction # so will use VGPR from consec elements? TODO - self.numVgprsPerDataPerVI = (1.0 * regsPerElement * kernelWriter.states.bpeCexternal) / kernelWriter.states.bpr + self.numVgprsPerDataPerVI = (1.0 * regsPerElement * bpeC) / kernelWriter.states.bpr elif beta: - self.numVgprsPerDataPerVI = (1.0 * kernelWriter.states.bpeCexternal) / kernelWriter.states.bpr + self.numVgprsPerDataPerVI = (1.0 * bpeC) / kernelWriter.states.bpr else: self.numVgprsPerDataPerVI = 0.0 @@ -101,7 +105,7 @@ def __init__(self, kernelWriter, kernel, ss, gwvw, edge, beta, atomic): self.halfDataRegPerVI = gwvw*self.numVgprsPerDataPerVI == 0.5 and not (kernel["ProblemType"]["UseInitialStridesCD"] and kernelWriter.states.archCaps["HasEccHalf"]) and not (kernel["ProblemType"]["DestDataType"].numRegisters() == 0.25) # StoreState constructor: - def __init__(self, kernelWriter, kernel, gwvw, edge, beta, atomic, elements): + def __init__(self, kernelWriter, kernel, gwvw, edge, beta, atomic, elements, isWorkspace=False): self.kernelWriter = kernelWriter self.kernel = kernel self.lsu = kernel["LocalSplitU"] @@ -154,7 +158,7 @@ def __init__(self, kernelWriter, kernel, gwvw, edge, beta, atomic, elements): if not atomic and len(kernel["PackedC1IndicesX"]) == 1: self.optSrdIncForRow = 1 - if kernel["StoreRemapVectorWidth"]: + if kernel["StoreRemapVectorWidth"] and not isWorkspace: self.optSrdIncForRow = 1 if kernel["ProblemType"]["UseInitialStridesCD"]: @@ -169,7 +173,7 @@ def __init__(self, kernelWriter, kernel, gwvw, edge, beta, atomic, elements): assert (not (self.optSingleColVgpr and self.optSharedColVgpr)) - self.cfg = self.StoreConstConfig(kernelWriter, kernel, self, gwvw, edge, beta, atomic) + self.cfg = self.StoreConstConfig(kernelWriter, kernel, self, gwvw, edge, beta, atomic, isWorkspace) # Use to detect new rows: self.lastCoordOffset1 = 0 @@ -244,9 +248,22 @@ def __init__(self, kernelWriter, kernel, gwvw, edge, beta, atomic, elements): # For detecting when we are running first batch self.firstBatch = True + # TODO code migrated from globalWrite, comments outdated with respect to features like activation + # how many vgprs are needed for zero elements + # 2 for addressC in vgpr for addition - already checked out + # 2 for coord0,1 of thread - already checked out + # 2 for tmp - already checked out + + # 5 = how many vgprs are needed per element (flat) + # - 2 for addr + # - 3 for GLOBAL_OFFSET_C calculation (can overlap below, therefore max) + # - if beta gwvw*rpe for new value + # - if atomic 2*rpe for old and cmp values + # Calculate numVgprsPerElement # print("numVgprsPerAddr=%u, numVgprsPerDataPerVI=%u, numVgprPerValuC=%u"%(self.cfg.numVgprsPerAddr, self.cfg.numVgprsPerDataPerVI, self.cfg.numVgprPerValuC)) self.numVgprsPerElement = self.cfg.numVgprPerValuC*gwvw + self.cfg.numVgprsPerAddr + int(ceil(self.cfg.numVgprsPerDataPerVI * gwvw)) + # TODO STREAM-K Update numVgprsPerElement for workspace if kernel["GroupLoadStore"] and kernel["ProblemType"]["UseBeta"]: self.numVgprsPerElement += self.cfg.numVgprsPerAddr if kernel["ProblemType"]["UseE"] and (kernel["GlobalSplitU"] == 1): @@ -347,7 +364,7 @@ def getStoreElementsInfoForBatch(self, kernel, batchElements): return self.elementCoord0, self.elementCoord1 - def setupStoreElementsForBatch(self, kernel, gwvw, batchElements, batchElementSgprs, isOptNLL, biasDim): + def setupStoreElementsForBatch(self, kernel, gwvw, batchElements, batchElementSgprs, isOptNLL, biasDim, isWorkspace=False): self.elementAddr = [] self.elementDataE = [] @@ -362,6 +379,9 @@ def setupStoreElementsForBatch(self, kernel, gwvw, batchElements, batchElementSg #generate elementCoord0/1 self.getStoreElementsInfoForBatch(kernel, batchElements) kw = self.kernelWriter + dataType = kernel["ProblemType"]["DestDataType"] + if isWorkspace: + dataType = kernel["ProblemType"]["ComputeDataType"] biasVgprMap = {} scaleAlphaVecVgprMap = {} @@ -460,7 +480,7 @@ def setupStoreElementsForBatch(self, kernel, gwvw, batchElements, batchElementSg if self.cfg.halfDataRegPerVI: # TODO- check (H,H,H,H,S,S) if kernel["ProblemType"]["HighPrecisionAccumulate"] and \ - (kernel["ProblemType"]["DestDataType"].isBFloat16() or kernel["ProblemType"]["DestDataType"].isHalf()): + (dataType.isBFloat16() or dataType.isHalf()): data = kw.vgprPool.checkOutAligned(int(2*self.cfg.numVgprsPerDataPerVI*self.cfg.gwvw), \ int(ceil(int(2*self.cfg.numVgprsPerDataPerVI*self.cfg.gwvw))), "writeBatch-data for ei=%u and ei=%u"%(elementIdx,elementIdx+1), preventOverflow=not isOptNLL) else: diff --git a/tensilelite/Tensile/Components/NotLocalFullTileElements.py b/tensilelite/Tensile/Components/NotLocalFullTileElements.py index 1376a48ce6..6cb789b832 100644 --- a/tensilelite/Tensile/Components/NotLocalFullTileElements.py +++ b/tensilelite/Tensile/Components/NotLocalFullTileElements.py @@ -28,22 +28,14 @@ class NotLocalFullTileElementsMFMA(NotLocalFullTileElements): kernel = {"EnableMatrixInstruction": True} - """ - Partition thread-tile into writeElements for store code - This function creates the writeElement mapping for full tiles - (ie non-edge cases) - """ - def __call__(self, writer, kernel, edge): - elements = [] - storeVectorWidth = 0 - + def getElements(self, writer, kernel, edge): # When singleBuffer/atomic is enabled. We will have 2 different type of store. # One is GSU=1 normal store and another is GSU>1 atomic store. # storeVectorWidth indicates the atomic store vectorWidth. # storeVectorWidth_1 indicates the normal store vectorWidth. # For non-atomic cases, these two are the same. - gsuBackup = kernel["GlobalSplitU"] - kernel["GlobalSplitU"] = 2 + elements = [] + storeVectorWidth = 0 if edge: storeVectorWidth = kernel["StoreVectorWidth"] if kernel["_VectorStore"] else 1 storeVectorWidth = min(storeVectorWidth, writer.maxGwvw(kernel), kernel["AssertFree0ElementMultiple"]) @@ -75,22 +67,22 @@ def __call__(self, writer, kernel, edge): element = (tt1, tt0, vc1, vc0) elements.append(element) - kernel["GlobalSplitU"] = 1 - elements_1 = [] - storeVectorWidth_1 = 0 - if edge: - storeVectorWidth_1 = kernel["StoreVectorWidth"] if kernel["_VectorStore"] else 1 - storeVectorWidth_1 = min(storeVectorWidth_1, writer.maxGwvw(kernel), kernel["AssertFree0ElementMultiple"]) - else: - storeVectorWidth_1 = kernel["StoreVectorWidth"] if kernel["_VectorStore"] else 1 - storeVectorWidth_1 = min(storeVectorWidth_1, writer.maxGwvw(kernel)) + return (storeVectorWidth, elements) - for tt1 in range(0, ceil(totalTT1//vectorWidth1)): - for vc1 in range(0, vectorWidth1): - for tt0 in range(0, ceil(totalTT0//vectorWidth0)): - for vc0 in range(0, vectorWidth0, storeVectorWidth_1): # note step by storeVectorWidth - element = (tt1, tt0, vc1, vc0) - elements_1.append(element) - kernel["GlobalSplitU"] = gsuBackup - - return (storeVectorWidth, elements, storeVectorWidth_1, elements_1) + """ + Partition thread-tile into writeElements for store code + This function creates the writeElement mapping for full tiles + (ie non-edge cases) + """ + def __call__(self, writer, kernel, edge): + if kernel["GlobalSplitU"] == 0: + (storeVectorWidth, elements) = self.getElements(writer, kernel, edge) + return (storeVectorWidth, elements, storeVectorWidth, elements) + else: + gsuBackup = kernel["GlobalSplitU"] + kernel["GlobalSplitU"] = 2 + (storeVectorWidth, elements) = self.getElements(writer, kernel, edge) + kernel["GlobalSplitU"] = 1 + (storeVectorWidth_1, elements_1) = self.getElements(writer, kernel, edge) + kernel["GlobalSplitU"] = gsuBackup + return (storeVectorWidth, elements, storeVectorWidth_1, elements_1) diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index 80b058c8f6..24189b6994 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -23,10 +23,13 @@ from ..TensileInstructions import Module, Label, SAddU32, RegisterPoolResource, sgpr, scalarStaticDivideAndRemainder, \ SCmpLtU32, SCSelectB32, sMagicDivAlg2, SMulI32, SSubU32, SMinU32, SMovB32, SCBranchSCC1, SCmpLeU32, VMovB32, vgpr, \ SAddCU32, SCmpGtU32, SCMovB32, SAddI32, SCmpEQU32, SCBranchSCC0, SLShiftLeftB32, SLoadB32, SWaitCnt, SMEMModifiers, \ - log2 + log2, SBarrier, SStoreB32, SLongBranchPositive, SBranch, ceilDivide, replaceHolder, SNop, staticMultiply +from ..Common import print2 # from ..TensileInstructions.Containers import SMEMModifiers from ..Component import Component +from ..AsmStoreState import StoreState import abc +from copy import deepcopy class XCCMapping(Component): """ @@ -260,7 +263,7 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele skStoreLabel = Label(label=writer.labels.getNameInc("SK_Store"), comment="") # StreamK store branches - tmpSgpr = self.sgprPool.checkOut(4, "globalWriteElements", preventOverflow=0) + tmpSgpr = writer.sgprPool.checkOut(4, "globalWriteElements", preventOverflow=0) # if we did not start the tile, store partials # branch to beta == 0 store path module.add(SCmpEQU32(src0=sgpr("StreamKLocalStart"), src1=0, comment="does wg start tile?")) @@ -297,7 +300,7 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele # TODO FIXUP STEP!!!!!!!!!!!!!!! fixupEdge = [False] # Temporary hack to test no edge variant - kStr += self.fixupStep(kernel, vectorWidths, elements, fixupEdge, tmpVgpr, tmpCVTVgpr, sCtaIdx, skStoreLabel) + # kStr += self.fixupStep(kernel, vectorWidths, elements, fixupEdge, tmpVgpr, tmpCVTVgpr, sCtaIdx, skStoreLabel) if kernel["StreamK"] >= 2: sIterCount = writer.sgprPool.checkOut(1, "iterCount", preventOverflow=0) @@ -320,18 +323,18 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele return module @abc.abstractmethod - def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements): + def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel): pass - def writePartialsCommon(self, writer, kernel, skPartialsLabel, vectorWidths, elements): + def writePartialsCommon(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel): module = Module("StreamK Common writePartials") # No partials for atomic mode if kernel["StreamKAtomic"]: return module - module.add(skPartialsLabel.getLabelName()) - if kernel["DebugStreamK"] & 2 == 0: + module.add(skPartialsLabel) + if kernel["DebugStreamK"] & 2 != 0: return module # fixupEdge = [False] # Temporary hack to test no edge variant @@ -339,7 +342,7 @@ def writePartialsCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele partialsLabels = {} for edge in edges: - partialsLabels[edge] = Label(writer.labels.getNameInc("GW_Partials_E%u" % ( 1 if edge else 0), "")) + partialsLabels[edge] = Label(writer.labels.getNameInc("GW_Partials_E%u" % ( 1 if edge else 0)), comment="") if False in edges and True in edges: with self.allocTmpSgpr(4) as tmpSgprInfo: @@ -347,15 +350,11 @@ def writePartialsCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele for edge in edges: module.add(partialsLabels[edge]) - kStr += self.computeWorkspaceSrd(writer, kernel, sgpr("StreamKIdx")) - kStr += self.partialsWriteProcedure(kernel, vectorWidths, elements, False, False, edge, atomic, tmpVgpr, tmpCVTVgpr, isOptNLL, endLabel) + module.add(self.computeWorkspaceSrd(writer, kernel, sgpr("StreamKIdx"))) + module.add(self.partialsWriteProcedure(writer, kernel, vectorWidths, elements, False, False, edge, tmpVgpr, cvtVgprStruct, endLabel)) return module - - ############################################################################## - # Workspace SRD for FixupStep - ############################################################################## def computeWorkspaceSrd(self, writer, kernel, sCtaIdx, tmpSgpr = None): module = Module("StreamK Common computeWorkspaceSrd") @@ -380,6 +379,450 @@ def computeWorkspaceSrd(self, writer, kernel, sCtaIdx, tmpSgpr = None): writer.sgprPool.checkIn(tmpLocal) return module + + def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, beta, edge, tmpVgpr, cvtVgprStruct, endLabel): + module = Module("StreamK Common partialsWriteProcedure") + + # PreLoopVmcntCaseStr = "" + # # not generate Case 2 if StoreCInUnroll with StoreVectorWidth==1 (Case 2 will be same as Case 3) + # if self.canOptimizePreLoopLWVmcnt: + # if beta: + # self.currPreLoopVmcntCase = PreLoopVmcntCase.OrdNLL_B1_Store + # elif edge or (kernel["StoreCInUnroll"] and kernel["StoreVectorWidth"]==1): + # self.currPreLoopVmcntCase = PreLoopVmcntCase.OrdNLL_E1_Store + # else: + # self.currPreLoopVmcntCase = PreLoopVmcntCase.OptNLL_Store + # PreLoopVmcntCaseStr = inst("s_mov_b32", sgpr("PreLoopLWVmcntCase"), hex(self.currPreLoopVmcntCase.value), \ + # "for optimizing next PreLoop LW vmcnt, set to Case%u"%self.currPreLoopVmcntCase.value) + # # reset vmcnt if the dict has this key (OptNLL_Store, OrdNLL_E1_Store), + # # OrdNLL_B1_Store is excluded + # if self.currPreLoopVmcntCase in self.preLoopVmcntDict: + # self.preLoopVmcntDict[self.currPreLoopVmcntCase] = 0 + + edgeI = edge + #edgeI = True # set to True to disable vector stores + gwvw = vectorWidths[edgeI] + #print "globalWriteElements: edge=", edge, "beta=", beta, "atomic=", atomic + + ######################################## + # Calculate Vgprs for Write Batching + ######################################## + + ss = StoreState(writer, kernel, gwvw, edge, beta, False, elements[edgeI], isWorkspace=True) + + #print self.vgprPool.state() + # Use VGPR up to next occupancy threshold: + maxVgprs = writer.getMaxRegsForOccupancy(kernel["NumThreads"], writer.vgprPool.size(), \ + writer.getLdsSize(kernel), writer.agprPool.size(), writer.states.doubleVgpr) + if writer.states.serializedStore: # get aggressive when serializedStore is on; not necessarily exclusive to this parameter + # len(elements[edgeI]) + # tl = [] + # for i in range(self.vgprPool.size()-self.vgprPool.available(), maxVgprs): + # tl.append(self.vgprPool.checkOut(1, "grow-pool up to next occupancy for GlobalWrite")) + # for t in tl: + # self.vgprPool.checkIn(t) + writer.vgprPool.growPool(writer.vgprPool.size()-writer.vgprPool.available(), maxVgprs, 1, \ + "grow-pool up to next occupancy for GlobalWrite") + # align = 1 + # # align adjustment + # if self.ss.cfg.numVgprsPerAddr > 1: + # align = max(align, self.ss.cfg.numVgprsPerAddr) + # if self.ss.cfg.numVgprPerValuC*gwvw > 1: + # align = max(align, self.ss.cfg.numVgprPerValuC*gwvw) + # if int(ceil(self.ss.cfg.numVgprsPerDataPerVI * gwvw)) > 1: + # align = max(align, int(ceil(self.ss.cfg.numVgprsPerDataPerVI * gwvw))) + numVgprAvailable = writer.vgprPool.availableBlock(ss.numVgprsPerElement, ss.align) + + # Grow the register pool if needed - we need enough regs for at least one element + # Unfortunate since this means the write logic is setting the VGPR requirement + # for the entire kernel but at least we have a functional kernel. + # Before growing the pool, see if we can shrink the write vector width instead? + # TODO : the vgprSerial is needed for-ever and if we grow here will split the + # range of the tmps. Maybe want to move vgprSerial to first vgpr? + + # TODO: Minimum elems for StoreRemap + # TODO: Which of DataType or DestDataType is in a better sense? 0114: Check Using DestDataType + HSS + minElements = 1 + if kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16(): + minElements = 2 + elif kernel["ProblemType"]["DataType"].is8bitFloat(): + # TODO STREAM-K check if needed + minElements = 4 + minNeeded = minElements * ss.numVgprsPerElement + + shrinkDb = 0 + if shrinkDb: + print("numVgprAvailable=", numVgprAvailable, "minElements=", minElements, "minNeeded=", minNeeded) + + if numVgprAvailable < minNeeded: + gwvwOrig = gwvw + currentOccupancy = writer.getOccupancy(kernel["NumThreads"], writer.getLdsSize(kernel), \ + writer.vgprPool.size(), writer.agprPool.size(), writer.states.doubleVgpr) + futureOccupancy = writer.getOccupancy(kernel["NumThreads"], writer.getLdsSize(kernel), \ + writer.vgprPool.size() - numVgprAvailable + minNeeded, writer.agprPool.size(), writer.states.doubleVgpr) + + if shrinkDb: + print("currentOccupancy=%u futureOccupancy=%u VGPRs=%u numVgprAvail=%u vgprPerElem=%u" \ + % (currentOccupancy, futureOccupancy, writer.vgprPool.size(), \ + numVgprAvailable, minElements*ss.numVgprsPerElement)) + if futureOccupancy > currentOccupancy: + if shrinkDb: + print("warning: %s growing VGPR for GlobalWrite batching - this may bloat VGPR usage" % \ + (writer.states.kernelName)) + print(" numVgprAvailable=", numVgprAvailable, \ + "numVgprsPerElement=", ss.numVgprsPerElement, \ + "beta=", beta, "gwvw=", gwvw) + elif gwvw != gwvwOrig: + ss.gwvw = gwvw # make both representations consistent + if shrinkDb: + print2("info: %s shrank gwvw from %u to %u but kept occupancy same=%u." \ + % (writer.states.kernelName, gwvwOrig, gwvw, currentOccupancy)) + + if numVgprAvailable < minElements*ss.numVgprsPerElement: + print2("info: growing pool += %d * %d for GlobalWrite\n" \ + % (minElements,ss.numVgprsPerElement)) + print2(writer.vgprPool.state()) + # tl = [] + # for i in range(0,minElements): + # tl.append(self.vgprPool.checkOut(numVgprsPerElement, "grow-pool for GlobalWrite")) + # for t in tl: + # self.vgprPool.checkIn(t) + writer.vgprPool.growPool(0, minElements, ss.numVgprsPerElement, \ + "grow-pool for GlobalWrite") + numVgprAvailable = writer.vgprPool.available() + print2(writer.vgprPool.state()) + + # set atomicW after we potentially resize GWVW + # atomicW = min(gwvw, kernel["VectorAtomicWidth"]) + atomicW = min(gwvw, writer.getVectorAtomicWidth(kernel)) + + # print("NumVgprAvailable", numVgprAvailable) + if ss.numVgprsPerElement: + numElementsPerBatch = numVgprAvailable // ss.numVgprsPerElement + else: + numElementsPerBatch = len(elements[edgeI]) # max, do 'em all + + # assert(writer.states.numVgprValuC % gwvw == 0) # sanity check + + numElementsPerBatch = numElementsPerBatch if not kernel["NumElementsPerBatchStore"] else min(kernel["NumElementsPerBatchStore"],numElementsPerBatch) + + if shrinkDb: + print("NumElementsPerBatch=", numElementsPerBatch, "LimitedBySgprs=", ss.cfg.numElementsPerBatchLimitedBySgprs, \ + "WARNING" if ss.cfg.numElementsPerBatchLimitedBySgprs < numElementsPerBatch else "okay") + if ss.cfg.numElementsPerBatchLimitedBySgprs < numElementsPerBatch: + numElementsPerBatch = ss.cfg.numElementsPerBatchLimitedBySgprs + + # TODO: Which of DataType or DestDataType is in a better sense? 0114: Check Using DestDataType + HSS + if (kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16()): + # only do an even number of halves - since these share hi/lo pieces of some registers? + if numElementsPerBatch > 1: + numElementsPerBatch = int(numElementsPerBatch/2)*2 + elif not kernel["EnableMatrixInstruction"]: + # (excluding MFMA+LSU case. It can work without an issue) + # The globalWriteBatch routine below can't handle odd elements per batch + # and 0 elements per batch is illegal. + # so if we don't have *GPR resources to handle a larger batch then need + # to mark overflowedResources rather than generate a kernel that won't work. + # It might be possible to fix globalWriteBatch to handle this case but these + # are likely to be low-performing so likely not worth optimizing. + if shrinkDb: + print("WARNING: half requires at least two elements per batch") + self.overflowedResources = 3 + #elif kernel["ProblemType"]["DataType"].is8bitFloat(): + # if numElementsPerBatch > 1: + # numElementsPerBatch = int(numElementsPerBatch/4)*4 + + assert numElementsPerBatch > 0, "numElementsPerBatch=0 for %s"%writer.states.kernelName + + #numElementsPerBatch=min(2,numElementsPerBatch) # hack to control number of batches + # if atomic and (ss.optSingleColVgpr or ss.optSharedColVgpr): + # # hack to avoid re-using address vgpr across rows + # # atomics need to perform several memory operations + # # if the batch spans multiple rows, need multiple address vgpr + # # which is not currently supported in the two opt*ColVgpr modes + # firstRow = [e for e in elements[edgeI] if e[0]==0 and e[2]==0] + # numElementsPerBatch=min(len(firstRow),numElementsPerBatch) + + numBatches = max(1, ceilDivide(len(elements[edgeI]),numElementsPerBatch)) + + numSgprs = ss.cfg.numTempSgprPerBatch + ss.cfg.numMaskSgprPerBatch + ss.cfg.numMaskSgprPerElement * numElementsPerBatch + + # TODO STREAM-K activation code + + if writer.db["PrintStoreRegisterDb"]: + print("edgeI", edgeI, "NumBatches", numBatches, "NumElementsPerBatch", numElementsPerBatch, "numVgprsPerElement", ss.numVgprsPerElement, "len(elements[edgeI])", len(elements[edgeI])) + print("numSgprs=", numSgprs, "sgprPool.size()=", writer.sgprPool.size(), "numTempSgprPerBatch=", ss.cfg.numTempSgprPerBatch, + "numMaskSgprPerBatch=", ss.cfg.numMaskSgprPerBatch, "numMaskSgprPerElement=", ss.cfg.numMaskSgprPerElement) + print(writer.sgprPool.state()) + module.addComment1("edge=%d, allocate %u sgpr. perBatchTmpS=%u perBatchMaskS=%u perElementMaskS=%u elementsPerBatch=%u" % + (edgeI, numSgprs, ss.cfg.numTempSgprPerBatch, ss.cfg.numMaskSgprPerBatch, ss.cfg.numMaskSgprPerElement, numElementsPerBatch)) + #kStr += "// storeStats, %d, %d, %d\n"% (edgeI, numSgprs, numElementsPerBatch) + # so if we don't have *GPR resources to handle a larger batch then need + # to mark overflowedResources rather than generate a kernel that won't work. + with writer.allocTmpSgpr(numSgprs, 2) as tmpSgprRes: + tmpSgpr = tmpSgprRes.idx + elementSgprs = tmpSgpr + ss.cfg.numTempSgprPerBatch + + codeAccVgprRead = deepcopy(writer.codes.accVgprRead) if writer.states.serializedStore else None + # TODO STREAM-K remove this? + useCodeMulAlpha = kernel["MIArchVgpr"] and alpha and not (kernel["GlobalSplitU"] > 1) + if useCodeMulAlpha: # do not set codeAccVgprRead=None if GSU>1 + codeAccVgprRead = None + + for batchIdx in range(0, numBatches): + elementStartIdx = batchIdx * numElementsPerBatch + elementStopIdx = min(elementStartIdx + numElementsPerBatch, len(elements[edgeI])) + elementsThisBatch = elements[edgeI][elementStartIdx:elementStopIdx] + #print("BATCH[%u/%u]: elements[edgeI][%u:%u] VGPRs=%u" % (batchIdx, numBatches, elementStartIdx, elementStopIdx,numVgprsPerElement )) + # elementVgprs can be large and should be perfectly tuned to the number of available + # VGPRS. We do not want to accidentally overflow and grow the pool here: + + module.add(self.partialsWriteBatch(writer, kernel, ss, batchIdx, alpha, beta, edge, gwvw, atomicW, \ + elementsThisBatch, writer.vgprs.addrD, writer.vgprs.addrC, \ + tmpVgpr, cvtVgprStruct, \ + elementSgprs, tmpSgpr, codeAccVgprRead)) + # delay PreLoopVmcntCase code after globalWrite + # if self.canOptimizePreLoopLWVmcnt: + # kStr += PreLoopVmcntCaseStr + + # Set flag + module.add(SWaitCnt(vmcnt=0, comment="wait for data store")) + module.add(SBarrier(comment="store all data before setting flag")) + module.add(SLShiftLeftB32(dst=sgpr(tmpSgpr), src=sgpr("StreamKIdx"), shiftHex=log2(4), comment="flag offset based on CTA index")) + module.add(SMovB32(dst=sgpr(tmpSgpr+2), src=1, comment="flag data")) + module.add(SStoreB32(src=sgpr(tmpSgpr+2), base=sgpr("AddressFlags", 2), soffset=sgpr(tmpSgpr), smem=SMEMModifiers(glc=1), comment="set flag")) + module.add(SWaitCnt(lgkmcnt=0, comment="wait for flag")) # TODO just for testing + + # TODO - if this is the last tile, don't need to jump to next instruction + # NOTE: in SR kernel, we need long branch since PRNG explodes the line of codes + if kernel["ProblemType"]["StochasticRounding"]: # in-device RND + with self.allocTmpSgpr(3) as tmpSgprInfo: + module.add(SLongBranchPositive(endLabel, tmpSgprInfo)) + else: + module.add(SBranch(labelName=endLabel.getLabelName(), comment="jump to end")) + + # Finish one write path, reset currPreLoopVmcntCase to Undefined + # self.currPreLoopVmcntCase = PreLoopVmcntCase.Undefined + + return module + + def partialsWriteBatch(self, writer, kernel, ss, batchIdx, applyAlpha, beta, edge, gwvw, atomicW, \ + batchElements, addrD, addrC, \ + tmpVgpr, cvtVgprStruct, batchElementSgprs, tmpSgpr, codeAccVgprRead): + module = Module("StreamK Common partialsWriteBatch") + + module.addComment0("optSingleColVgpr=%u optSharedColVgpr=%u optSGPRUsage=%s optSrdIncForRow=%u" % \ + (ss.optSingleColVgpr, ss.optSharedColVgpr, ss.optSGPRUsage, ss.optSrdIncForRow)) + + if kernel["StoreSyncOpt"]: + module.add(SSleep(kernel["StoreSyncOpt"] - 1, "optimization: sync and wait")) + module.add(SBarrier()) + + # comment tt1, tt0, vc1, vc0 + # tt = thread tile, vc=vector component + commentStr = "Partials Write%s%s%s Batch #%u (d1,d0,vc1,vc0) =\n " \ + % (" Alpha" if applyAlpha else "", " Beta" if beta else "", " Edge" if edge else "", batchIdx) + for elementIdx in range(0, len(batchElements)): + element = batchElements[elementIdx] + commentStr += "(%u,%u,%u,%u:vw%u)" % (element[0], element[1], element[2], element[3], gwvw) + if elementIdx < len(batchElements)-1: + commentStr += "; " + module.addComment2(commentStr) + + # allow expanding vgpr pool for OptNLL + # preventOverflow = (not isOptNLL) + # ss.setupStoreElementsForBatch(kernel, gwvw, batchElements, batchElementSgprs, isOptNLL=isOptNLL, isWorkspace=True) + ss.setupStoreElementsForBatch(kernel, gwvw, batchElements, batchElementSgprs, isOptNLL=False, biasDim=0, isWorkspace=True) + + storesIssued = 0 + tmpS01 = tmpSgpr # scratch sgprs + + ######################################## + # calculate addr and masks + module.addComment1("calc coords, apply mask, and issue loads (if necessary)") + # On input, coord0 and coord1 are VGPRs computed in the pre-batch code, based + # on the thread and tid number. These are ELEMENT offsets from start of tensor C + # for the top-left corner this thread will write. These are not changed + # across all the store loop iters. + if writer.db["ConservativeWaitCnt"] & 0x10: + module.add(SBarrier("debug")) + module.add(SWaitCnt(vmcnt=0, comment="ConservativeWaitCnt")) + if writer.states.archCaps["SeparateVscnt"]: + module.add(SWaitCnt(vscnt=0, comment="writes")) + module.add(SBarrier("debug")) + if not edge and writer.db["ForceEdgeStores"]>=2: + module.add(self.parentWriter.getBomb()) # should not get here + if edge and writer.db["AssertNoEdge"]: + module.add(self.parentWriter.getBomb()) # should not get here + + ## create code Module to push mov vgpr,acc instructions + # if kernel["StoreCInUnroll"] and not edge: + # accVgprRead = Code.Module("movaccVgpr") + # self.StoreCUnrollLoadCWaitComment = "waitcnt for LoadC" # this will be used later to identify waitcnt for loadC + + ######################################## + # AccVgpr read + # if kernel.enabledSetPrioSplitLDS: + # kStr += inst("s_setprio", "0", "") + if codeAccVgprRead is not None: # and writer.kernel["LocalSplitU"] == 1 + regsPerScalar = writer.states.bpeCinternal // writer.states.bpr # register per scalar + # loop over store instructions within one batch + for elementIdx in range(0, len(batchElements)): + # loop over scalars within one store instruction + for vi in range(0, gwvw): + # loop over registers within one scalar + for rIdx in range(0, regsPerScalar): + module.add(replaceHolder(codeAccVgprRead.items().pop(0), ss.elementSumIdx[elementIdx]*regsPerScalar + regsPerScalar*vi + rIdx)) + # module.add(replaceHolder(self.codeAccVgprRead.items().pop(0), ss.elementSumIdx[elementIdx]*regsPerScalar + regsPerScalar*vi + rIdx - self.parentWriter.states.c.startVgprValu)) + # if kernel["StoreCInUnroll"] and not edge: + # tempStr = tempStr.replace("__placeholder__",str(elementIdx*gwvw*regsPerScalar + regsPerScalar*vi + rIdx)) + # accVgprRead.addCode(tempStr.replace("ValuC","L2GC")) + + if not kernel["MIArchVgpr"]: + module.add(SNop(1, "2 wait states required before reading vgpr")) + + ######################################## + # Not Atomic + ######################################## + # else: + # edge has v_cndmask so loads or stores may not issue, hard to track vmcnt: + for elementIdx in range(len(batchElements)): + for vi in range(gwvw): + sumIdxV = ss.elementSumIdx[elementIdx] + vi + # TODO STREAM-K is start value needed now? + # TODO KUPO!!!!!!!!!!!!!!!! + # newSumIdxV = sumIdxV - writer.states.c.startVgprValu + # covers sgemm, gemm_ex(HHS/HSS/BBS/BSS (HPA=T)), int8 (int8x4?) + if kernel["ProblemType"]["ComputeDataType"].isInt32() or \ + kernel["ProblemType"]["ComputeDataType"].isSingle(): # covers sgemm/gemm_ex(HHS/HSS/BBS/BSS) + if writer.db["ForceExpectedValue"]: + module.add(VMovB32(dst=vgpr("ValuC+%u"%sumIdxV), src=writer.db["ValueCExpectedValue"], comment="force expected value")) + # module.add(VMovB32(dst=vgpr("ValuC+%u"%newSumIdxV), src=self.debugConfig["ValueCExpectedValue"], comment="force expected value" )) + if writer.db["ForceVSerial"]: + module.add(VMovB32(dst=vgpr("ValuC+%u"%sumIdxV), src=vgpr("Serial"), comment="force expected value to serial")) + # module.add(VMovB32(dst=vgpr("ValuC+%u"%newSumIdxV), src=vgpr("Serial"), comment="force expected value to serial" )) + if writer.db["CheckValueC"]: + module.add(SMovB32(dst=sgpr(tmpS01), src=writer.db["ValueCExpectedValue"], comment="Move expected value")) + module.add(writer.getCmpAssert(writer.asmAssert.eq, vgpr("ValuC+%u"%sumIdxV), sgpr(tmpS01))) + # module.add(SMovB32(dst=sgpr(self.tmpS01), src=self.debugConfig["ValueCExpectedValue"], comment="Move expected value")) + # module.add(self.parentWriter.getCmpAssert(self.parentWriter.asmAssert.eq, vgpr("ValuC+%u"%newSumIdxV), sgpr(self.tmpS01))) + + module.addComment1("apply mask, calc new C and issue writes") + #kStr += self.bomb() # can see store addresses just before the store inst + + # if kernel["ProblemType"]["DestDataType"].isBFloat16() and kernel["ProblemType"]["HighPrecisionAccumulate"]: + # vgprBf16Temp = tmpCVTVgpr + # vgprBf16Mask = vgprBf16Temp + 1 + # vgprFp32Nan = vgprBf16Temp + 2 + # vgprBf16Inc = vgprBf16Temp + 3 + # kStr += inst("v_mov_b32", vgpr(vgprBf16Mask), "0xffff0000", "mask for pack two bfloat16 element to 32bit" ) + # kStr += inst("v_mov_b32", vgpr(vgprFp32Nan), "0x7fff0000", "fp32 Nan" ) + # kStr += inst("v_mov_b32", vgpr(vgprBf16Inc), "0x7fff", "rounding bias for bfloat16" ) + if kernel["ProblemType"]["DestDataType"].isBFloat16() and kernel["ProblemType"]["HighPrecisionAccumulate"]: + module.add(VMovB32(vgpr(cvtVgprStruct.vgprBf16Mask), "0xffff0000", "mask for pack two bfloat16 element to 32bit" )) + module.add(VMovB32(vgpr(cvtVgprStruct.vgprFp32Nan), "0x7fff0000", "fp32 Nan" )) + module.add(VMovB32(vgpr(cvtVgprStruct.vgprBf16Inc), "0x7fff", "rounding bias for bfloat16" )) + elif kernel["ProblemType"]["DestDataType"].isFloat8() and kernel["ProblemType"]["HighPrecisionAccumulate"]: + module.add(VMovB32(vgpr(cvtVgprStruct.vgprFp8NanInf), "0x207", "Nan and +/- inf" )) + module.add(VMovB32(vgpr(cvtVgprStruct.vgprFp8Max), "0x43700000", "Fp8 Max value 240 as float32" )) + module.add(VMovB32(vgpr(cvtVgprStruct.vgprFp8Min), "0xc3700000", "Fp8 Min value -240 as float32" )) + elif kernel["ProblemType"]["DestDataType"].isBFloat8() and kernel["ProblemType"]["HighPrecisionAccumulate"]: + module.add(VMovB32(vgpr(cvtVgprStruct.vgprBF8NanInf), "0x207", "Nan and +/- inf" )) + module.add(VMovB32(vgpr(cvtVgprStruct.vgprBF8Max), "0x47600000", "BF8 Max value 57344 as float32" )) + module.add(VMovB32(vgpr(cvtVgprStruct.vgprBF8Min), "0xc7600000", "BF8 Min value -57344 as float32" )) + + # DestDataType for 8bit Float can only be F8 or B8 + # if kernel["ProblemType"]["DestDataType"].isFloat8() or kernel["ProblemType"]["DestDataType"].isBFloat8(): # F8 is always HPA + # # make vgprF8Temp0 always even to use pk instruction later + # if tmpCVTVgpr % 2 == 0: + # vgprF8Temp0 = tmpCVTVgpr + # vgprF8Max = vgprF8Temp0 + 2 + # vgprF8Min = vgprF8Temp0 + 3 + # else: + # vgprF8Max = tmpCVTVgpr + # vgprF8Temp0 = vgprF8Max + 1 + # vgprF8Min = vgprF8Max + 3 + + # if kernel["ProblemType"]["Fp32toFp8SWClip"]: + # # set flag of f32 NaN and +/- INF for v_cmp_class + # vgprFp32NanInfFlag = vgprF8Min + 1 + # kStr += inst("v_mov_b32", vgpr(vgprFp32NanInfFlag), "0x207", "flag for Nan and +/- inf" ) + # # set max/min values for clipping + # if kernel["ProblemType"]["DestDataType"].isFloat8(): + # kStr += inst("v_mov_b32", vgpr(vgprF8Max), "0x43700000", "save 240.0f as max for clipping" ) + # kStr += inst("v_mov_b32", vgpr(vgprF8Min), "0xC3700000", "save -240.0f as min for clipping" ) + # else: #BFloat8 + # kStr += inst("v_mov_b32", vgpr(vgprF8Max), "0x47600000", "save 57344.0f as max for clipping" ) + # kStr += inst("v_mov_b32", vgpr(vgprF8Min), "0xC7600000", "save -57344`.0f as min for clipping" ) + + storeCode = Module("Partials GroupLoadStore") + for elementIdx in range(len(batchElements)): + element = batchElements[elementIdx] + addrCalc: AddrCalculation = ss.elementAddr[elementIdx] + addr = addrCalc.addrDVgpr + sumIdx = ss.elementSumIdx[elementIdx] + + storeWidth = kernel["StoreVectorWidth"] + # storeWidth = 2 + if batchIdx == 0 and elementIdx == 0: + tmpSgprRes = RegisterPoolResource(idx=tmpS01, size=1) + module.add(staticMultiply(vgpr(addr), vgpr("Serial"), storeWidth * writer.states.bpeCinternal, tmpSgprRes)) + # kStr += inst("v_mul_lo_u32", , "Partials buffer address") + module.add(SMovB32(dst=sgpr(tmpS01), src=0, comment="Init sgpr offset")) + else: + increment = (kernel["WavefrontSize"] * 4) * storeWidth * writer.states.bpeCinternal + module.add(SAddU32(dst=sgpr(tmpS01), src0=sgpr(tmpS01), src1=increment, comment="Inc sgpr offset")) + + # TODO StreamK need this packing code??? + # if self.asmCaps["HasWMMA"] and kernel["EnableMatrixInstructionStore"] and kernel["ProblemType"]["DestDataType"].isHalf() and (not kernel["ProblemType"]["HighPrecisionAccumulate"]): + # for vi in range(0, gwvw): + # sumIdxV = ss.elementSumIdx[elementIdx] + vi + # if vi%2 == 1: + # d = ss.elementSumIdx[elementIdx] + vi//2 + # kStr += inst("v_pack_b32_f16", vgpr(d), vgpr("ValuC+%u"%(sumIdxV-1)), vgpr("ValuC+%u"%sumIdxV), "Pack with neighbor" ) + + # if not kernel["StoreRemapVectorWidth"]: + tmpStoreCode = writer.addStore(kernel, ss, 'WS', addrCalc, sumIdx, tmpS01, edge, wsOffset=sgpr(tmpS01)) + if kernel["GroupLoadStore"]: + storeCode.add(tmpStoreCode) + else: + module.add(tmpStoreCode) + storesIssued += 1 + + module.add(storeCode) + + # return registers to pool: + lastData = -1 + for elementIdx in range(0, len(batchElements)): + if not ss.sharedColDVgprs: + addrCalc: AddrCalculation = ss.elementAddres[elementIdx] + addrDVgpr = addrCalc.addrDVgpr + addrCVgpr = addrCalc.addrCVgpr + writer.vgprPool.checkIn(addrDVgpr) + if addrCVgpr != addrDVgpr: + writer.vgprPool.checkIn(addrCVgpr) + + data = ss.elementData[elementIdx] + if data != 0: + if data != lastData: + writer.vgprPool.checkIn(data) + lastData = data + + ss.firstBatch = False + ss.checkInTempVgprC() + + if writer.states.serializedStore: + module.add(SNop(0, "1 wait state required when next inst writes vgprs held by previous dwordx4 store inst")) + + # Update the store cnt to preLoopVmcntDict for Case2/3 + # (No need to update for Case0:'Undefined' or Case4:'OrdNLL_B1_Store') + # TODO STREAM-K Need this? + # if self.currPreLoopVmcntCase in self.preLoopVmcntDict: + # if not self.archCaps["SeparateVscnt"]: + # self.preLoopVmcntDict[self.currPreLoopVmcntCase] += storesIssued + + return module + class StreamKOff(StreamK): kernel = {"StreamK": 0} @@ -433,7 +876,7 @@ def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements) module = Module("StreamK Off storeBranches") return module - def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements): + def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel): module = Module("StreamK Off writePartials") return module @@ -504,9 +947,9 @@ def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements) module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) return module - def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements): + def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel): module = Module("StreamK Basic writePartials") - module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) + module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel)) return module class StreamKTwoTileOriginal(StreamK): @@ -614,9 +1057,9 @@ def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements) module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) return module - def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements): + def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel): module = Module("StreamK TwoTileOriginal writePartials") - module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) + module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel)) return module @@ -760,7 +1203,7 @@ def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements) module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) return module - def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements): + def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel): module = Module("StreamK TwoTileDPFirst writePartials") - module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) + module.add(self.writePartialsCommon(writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel)) return module diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index e523aee654..2ac36d1802 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -3640,6 +3640,16 @@ def readWriteVectors(mat, vw, kernel): if kernel["GlobalSplitU"] > 0: self.defineSgpr("GSU", 1) # Can't move to the front because of the preload arguments + + if kernel["StreamK"]: + # StreamK vars + self.defineSgpr("StreamKIdx", 1) + self.defineSgpr("StreamKIter", 1) + self.defineSgpr("StreamKIterEnd", 1) + self.defineSgpr("StreamKLocalStart", 1) + self.defineSgpr("StreamKLocalEnd", 1) + if kernel["StreamKAtomic"] == 0: + self.defineSgpr("SrdWS", 4, 4) #------------------------ # Registers defined below this point are not available in the post-loop diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 0f80368a5c..44b2e76a9d 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -389,16 +389,6 @@ def defineVariableSgprs(self, kernel): # Mostly impacts flat kernels and GSU edge since these need SGPR # for conditionals - if kernel["StreamK"]: - # StreamK vars - module.add(self.defineSgpr("StreamKIdx", 1)) - module.add(self.defineSgpr("StreamKIter", 1)) - module.add(self.defineSgpr("StreamKIterEnd", 1)) - module.add(self.defineSgpr("StreamKLocalStart", 1)) - module.add(self.defineSgpr("StreamKLocalEnd", 1)) - if kernel["StreamKAtomic"] == 0: - module.add(self.defineSgpr("SrdWS", 4, 4)) - if kernel["BufferLoad"]: # resource descriptor (SRD) A and B, must be aligned on 4-SGPR boundary module.add(self.defineSgpr("SrdA", 4, 4)) @@ -8885,7 +8875,7 @@ def findInstCount(module, targetItem, count): self.sgprPool.checkIn(activationSetPCStruct.sgprOffsetActivation) self.sgprPool.checkIn(activationSetPCStruct.sgprOffsetBack) - module.add(skComponent.writePartials(self, kernel, skPartialsLabel, vectorWidths_1, elements_1)) + module.add(skComponent.writePartials(self, kernel, skPartialsLabel, vectorWidths_1, elements_1, tmpVgpr, cvtVgprStruct, endLabel)) # End label module.add(endLabel) @@ -8934,23 +8924,12 @@ def globalWriteElementBatch(self, kernel, tPA, tPB, activation, \ ss = StoreState(self, kernel, gwvw, edge, beta, atomic, elements[edgeI]) - # how many vgprs are needed for zero elements - # 2 for addressC in vgpr for addition - already checked out - # 2 for coord0,1 of thread - already checked out - # 2 for tmp - already checked out - - # 5 = how many vgprs are needed per element (flat) - # - 2 for addr - # - 3 for GLOBAL_OFFSET_C calculation (can overlap below, therefore max) - # - if beta gwvw*rpe for new value - # - if atomic 2*rpe for old and cmp values - #print self.vgprPool.state() # Use VGPR up to next occupancy threshold: maxVgprs = self.getMaxRegsForOccupancy(kernel["NumThreads"], self.vgprPool.size(), \ self.getLdsSize(kernel), self.agprPool.size(), self.states.doubleVgpr) if self.states.serializedStore: # get aggressive when serializedStore is on; not necessarily exclusive to this parameter - _growPool(self.vgprPool, self.vgprPool.size()-self.vgprPool.available(), maxVgprs, 1, \ + self.vgprPool.growPool(self.vgprPool.size()-self.vgprPool.available(), maxVgprs, 1, \ "grow-pool up to next occupancy for GlobalWrite") # Get numVgprAvailable numVgprAvailable = self.vgprPool.availableBlock(ss.numVgprsPerElement, ss.align) @@ -8997,7 +8976,7 @@ def globalWriteElementBatch(self, kernel, tPA, tPB, activation, \ print2("info: growing pool += %d * %d for GlobalWrite\n" \ % (minElements,ss.numVgprsPerElement)) print2(self.vgprPool.state()) - _growPool(self.vgprPool, 0, minElements, ss.numVgprsPerElement, \ + self.vgprPool.growPool(0, minElements, ss.numVgprsPerElement, \ "grow-pool for GlobalWrite") numVgprAvailable = self.vgprPool.available() print2(self.vgprPool.state()) @@ -9281,7 +9260,8 @@ def bufferLoadImpl(soffset): ############################################################################## def chooseGlobalWrite(self, useBuffer, bps, srcVgpr, rpv, \ - addr0, addr1, offset, glc=False, slc=False, nt=False, hi16=0, comment="store"): + addr0, addr1, offset, soffset=0, \ + glc=False, slc=False, nt=False, hi16=0, comment="store"): """ create the store instruction for requested vector width and other parms rpv = regs per vector @@ -9334,6 +9314,8 @@ def bufferStoreImpl(tmpSgpr, mubuf): if useBuffer: mubuf = MUBUFModifiers(offen=True, offset12=offset, glc=glc, slc=slc, nt=nt, isStore=True) + if soffset != 0: + assert offset < 4096, "sgpr offset provided with large const offset" # buffer_load offset field is 12-bit. # if offset >= 4096, use soffset instead maxShift = max(bps - 16, 0) #if bps = 32 or bps = 64 @@ -9346,7 +9328,7 @@ def bufferStoreImpl(tmpSgpr, mubuf): mubuf.offset12 = 0 bufferStoreImpl(tmpSgpr, mubuf) else: - bufferStoreImpl(0, mubuf) + bufferStoreImpl(soffset, mubuf) else: flat = FLATModifiers(glc=glc, slc=slc, isStore=True) @@ -9456,7 +9438,7 @@ def addBiasLoad(self, dataType, kernel, gwvw, addrCalc, biasVgpr, biasDim, isLoc return self.addBiasGlobalLoad(dataType, kernel, biasVgpr, addr0, addr1, addrCalc.biasOffset[biasDim], gwvw) ############################################################################## - def addStore(self, kernel, ss, tc: str, addrCalc, sumIdx, tmpS01, edge, comment): + def addStore(self, kernel, ss, tc: str, addrCalc, sumIdx, tmpS01, edge, wsOffset=0, comment="addStore"): """ Add stores for the element with addrCalc and sumIdx. tmpS01 is a single :temp sGPR @@ -9508,6 +9490,21 @@ def addStore(self, kernel, ss, tc: str, addrCalc, sumIdx, tmpS01, edge, comment) dataType = kernel["ProblemType"]["DestDataType"] globalOffset = addrCalc.globalOffset globalOffset = int((globalOffset/self.states.bpeCexternal) * self.states.bpr * kernel["ProblemType"]["DestDataType"].numRegisters()) + elif tc == 'WS': + isGlc = True + isSlc = True + isNT = kernel["NonTemporalD"] & 0x4 + + bps = self.states.bpeCinternal * ss.cfg.gwvw + rpv = self.states.bpeCinternal * ss.cfg.gwvw / self.states.bpr + if kernel["BufferStore"]: + addr0 = vgpr(addrCalc.addrDVgpr) + addr1 = sgpr("SrdWS", 4) + else: + addr0 = vgpr(addrCalc.addrDVgpr,2) + addr1 = "" + dataType = kernel["ProblemType"]["ComputeDataType"] + globalOffset = 0 elif tc == 'Bias': bps = self.states.bpeCinternal * ss.cfg.gwvw rpv = self.states.bpeCinternal * ss.cfg.gwvw / self.states.bpr @@ -9545,30 +9542,37 @@ def addStore(self, kernel, ss, tc: str, addrCalc, sumIdx, tmpS01, edge, comment) # (H,H,H,H,H,H), internal H if self.states.asmCaps["HasWMMA"] and kernel["EnableMatrixInstruction"]: module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx, rpv, \ - addr0, addr1, globalOffset, isGlc, isSlc, isNT, hi16=0, comment=comment)) + addr0, addr1, globalOffset, soffset=wsOffset, \ + glc=isGlc, slc=isSlc, nt=isNT, hi16=0, comment=comment)) else: module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx//2, rpv, \ - addr0, addr1, globalOffset, isGlc, isSlc, isNT, hi16=sumIdx%2, comment=comment)) + addr0, addr1, globalOffset, soffset=wsOffset, \ + glc=isGlc, slc=isSlc, nt=isNT, hi16=sumIdx%2, comment=comment)) else: # (B,B,B,B,S,S), internal S # (H,H,H,H,H,H), internal S # (H,H,H,H,S,S), internal S module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx, rpv, \ - addr0, addr1, globalOffset, isGlc, isSlc, isNT, hi16=0, comment=comment)) + addr0, addr1, globalOffset, soffset=wsOffset, \ + glc=isGlc, slc=isSlc, nt=isNT, hi16=0, comment=comment)) elif dataType.isInt32() or dataType.isSingle(): module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx, rpv, \ - addr0, addr1, globalOffset, isGlc, isSlc, isNT, comment=comment)) + addr0, addr1, globalOffset, soffset=wsOffset, \ + glc=isGlc, slc=isSlc, nt=isNT, comment=comment)) elif dataType.isDouble() or dataType.isSingleComplex(): module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx*2, rpv, \ - addr0, addr1, globalOffset, isGlc, isSlc, isNT, comment=comment)) + addr0, addr1, globalOffset, soffset=wsOffset, \ + glc=isGlc, slc=isSlc, nt=isNT, comment=comment)) elif dataType.isDoubleComplex(): rps = dataType.numRegisters() module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx*rps, rpv, \ - addr0, addr1, globalOffset, isGlc, isSlc, isNT, comment=comment)) + addr0, addr1, globalOffset, soffset=wsOffset, \ + glc=isGlc, slc=isSlc, nt=isNT, comment=comment)) elif dataType.isInt8() or dataType.isFloat8() or dataType.isBFloat8() or dataType.isFloat8BFloat8() or dataType.isBFloat8Float8(): if kernel["ProblemType"]["HighPrecisionAccumulate"]: module.add(self.chooseGlobalWrite(useBuffer, bps, sumIdx, rpv, \ - addr0, addr1, globalOffset, isGlc, isSlc, isNT, comment=comment)) + addr0, addr1, globalOffset, soffset=wsOffset, \ + glc=isGlc, slc=isSlc, nt=isNT, comment=comment)) return module ############################################################################## @@ -10341,10 +10345,3 @@ def _getEccOffset(totalWidth, bpr, bpe, glvw, idx, numVgprG2L): return numVgprG2L * left else: return 0 - -def _growPool(pool: RegisterPool, rangeStart: int, rangeEnd: int, checkOutSize: int, comment: str=""): - tl = [] - for _ in range(rangeStart, rangeEnd): - tl.append(pool.checkOut(checkOutSize, comment)) - for t in tl: - pool.checkIn(t) diff --git a/tensilelite/Tensile/TensileInstructions/RegisterPool.py b/tensilelite/Tensile/TensileInstructions/RegisterPool.py index 08bf707566..c003bce8e2 100644 --- a/tensilelite/Tensile/TensileInstructions/RegisterPool.py +++ b/tensilelite/Tensile/TensileInstructions/RegisterPool.py @@ -341,6 +341,13 @@ def state(self): def stateDetailed(self): for index, register in enumerate(self.pool): print("%u: %s"%(index, register.tag)) + + def growPool(self, rangeStart: int, rangeEnd: int, checkOutSize: int, comment: str=""): + tl = [] + for _ in range(rangeStart, rangeEnd): + tl.append(self.checkOut(checkOutSize, comment)) + for t in tl: + self.checkIn(t) @dataclass class RegisterPoolResource: From a100b1f7683a9c1f0bf27950bd9717829aa5a759 Mon Sep 17 00:00:00 2001 From: adityalj <167943392+adityalj@users.noreply.github.com> Date: Mon, 22 Jul 2024 15:07:47 -0400 Subject: [PATCH 15/36] Grid prediction (#4) * Analytical grid size prediction model for Stream-K --- .../amd_detail/rocblaslt/src/tensile_host.cpp | 42 ++- .../Source/client/source/SolutionIterator.cpp | 2 +- .../include/Tensile/ContractionSolution.hpp | 13 +- .../Source/lib/include/Tensile/Debug.hpp | 2 + .../Tensile/GranularitySelectionLibrary.hpp | 4 +- .../include/Tensile/MasterSolutionLibrary.hpp | 6 +- .../include/Tensile/SingleSolutionLibrary.hpp | 8 +- .../lib/include/Tensile/SolutionLibrary.hpp | 6 + .../Source/lib/source/ContractionSolution.cpp | 339 ++++++++++++++---- .../Tensile/Source/lib/source/Debug.cpp | 5 + 10 files changed, 331 insertions(+), 96 deletions(-) diff --git a/library/src/amd_detail/rocblaslt/src/tensile_host.cpp b/library/src/amd_detail/rocblaslt/src/tensile_host.cpp index 47b10938e7..7fba0580c5 100644 --- a/library/src/amd_detail/rocblaslt/src/tensile_host.cpp +++ b/library/src/amd_detail/rocblaslt/src/tensile_host.cpp @@ -1694,7 +1694,7 @@ rocblaslt_status makeArgument(rocblaslt_handle handle, if(useUserArgs) { data->kernels = solution->solveGroupedGemmGPU( - data->problem.gemms, data->inputs, nullptr, workspace, stream); + data->problem.gemms, data->inputs, *hardware, nullptr, workspace, stream); } else { @@ -1835,7 +1835,7 @@ rocblaslt_status getDeviceUserArgumentsValuesFromContractionProblem(rocblaslt_ha { std::shared_ptr data = std::static_pointer_cast(gemmData); - auto solution = library->getSolutionByIndex(data->algoIndex); + auto solution = library->getSolutionByIndex(*hardware, data->algoIndex); auto& problem = data->problem.gemms[0]; if(problem.activationComputeType() == Tensile::DataType::Float) { @@ -1896,7 +1896,7 @@ rocblaslt_status runKernelFromNewDeviceUserArguments(rocblaslt_handle hand for(auto& it : data->kernels) { uint8_t* arg = it.args.rawdata(); - auto solution = library->getSolutionByIndex(data->algoIndex); + auto solution = library->getSolutionByIndex(*hardware, data->algoIndex); if(solution->internalArgsSupport.useUniversalArgs) { if(deviceUserArgs != nullptr) @@ -1963,11 +1963,11 @@ rocblaslt_status runKernelFromDeviceUserArguments(rocblaslt_handle h // don't overwrite data->algoIndex = *solutionIndex; here if(gemmType == rocblaslt::RocGemmType::ROCBLASLT_GROUPED_GEMM) { - auto solution = library->getSolutionByIndex(*solutionIndex); + auto solution = library->getSolutionByIndex(*hardware, *solutionIndex); std::shared_ptr data = std::static_pointer_cast(gemmData); auto kernel = solution->solveGroupedGemmGPU( - data->problem.gemms, data->inputs, deviceUserArgs, workspace, stream); + data->problem.gemms, data->inputs,*hardware, deviceUserArgs, workspace, stream); status = hip2RocStatus(adapter->launchKernels(kernel, stream, nullptr, nullptr)); } else @@ -2007,6 +2007,7 @@ void _convertToHeuristicResultArray( int* returnAlgoCount, size_t maxWorkSpaceBytes, const Tensile::ContractionProblemGemm& problem, + const Tensile::Hardware& hardware, size_t fallbackCount) { *returnAlgoCount = std::min((int)solutions.size(), requestedAlgoCount); @@ -2019,7 +2020,7 @@ void _convertToHeuristicResultArray( heuristicResultsArray[i].algo.max_workspace_bytes = maxWorkSpaceBytes; heuristicResultsArray[i].algo.fallback = fallbackCount-- > 0 ? true : false; heuristicResultsArray[i].state = rocblaslt_status_success; - heuristicResultsArray[i].workspaceSize = solution->requiredWorkspaceSize(problem); + heuristicResultsArray[i].workspaceSize = solution->requiredWorkspaceSize(problem, hardware); } for(size_t i = *returnAlgoCount; i < requestedAlgoCount; i++) { @@ -2177,6 +2178,7 @@ rocblaslt_status getBestSolutions(RocblasltContractionProblem const& prob, returnAlgoCount, maxWorkSpaceBytes, data->problem, + *hardware, fallbackSize); return rocblaslt_status_success; @@ -2248,7 +2250,7 @@ rocblaslt_status getAllSolutions(MyProblem& heuristicResults[i].algo.fallback = false; heuristicResults[i].state = rocblaslt_status_success; if constexpr(std::is_same::value) - heuristicResults[i].workspaceSize = solution->requiredWorkspaceSize(prob); + heuristicResults[i].workspaceSize = solution->requiredWorkspaceSize(prob, *hardware); else heuristicResults[i].workspaceSize = 0; i++; @@ -2326,7 +2328,7 @@ rocblaslt_status for(auto index : solutionIndex) { isOutOfBound = isOutOfBound && (index > lastSolutionIndex); - auto solution = library->getSolutionByIndex(index); + auto solution = library->getSolutionByIndex(*hardware, index); if(!solution) continue; rocblaslt_matmul_heuristic_result result; @@ -2444,7 +2446,7 @@ rocblaslt_status isSolutionSupported(rocblaslt_handle handle, bool isSup = (*solution->hardwarePredicate)(*hardware) && (*solution->problemPredicate)(tensile_prob); if(isSup) - *workspaceSizeInBytes = solution->requiredWorkspaceSize(tensile_prob); + *workspaceSizeInBytes = solution->requiredWorkspaceSize(tensile_prob, *hardware); tensile_prob.setUseBias(useBias); tensile_prob.setActivationType(actType); tensile_prob.setUseScaleAlphaVec(useScaleAlphaVec); @@ -2474,7 +2476,7 @@ rocblaslt_status isSolutionSupported(rocblaslt_handle handle, } else { - *workspaceSizeInBytes = solution->requiredWorkspaceSize(tensile_prob); + *workspaceSizeInBytes = solution->requiredWorkspaceSize(tensile_prob, *hardware); } } else if constexpr(std::is_same::value) @@ -2509,7 +2511,7 @@ rocblaslt_status isSolutionSupported(rocblaslt_handle handle, bool isSupported = true; bool isNormalGemm = true; - auto problemWs = solution->requiredWorkspaceSizeGroupedGemm(tensile_prob.gemms); + auto problemWs = solution->requiredWorkspaceSizeGroupedGemm(tensile_prob.gemms, *hardware); for(int i = 0; i < tensile_prob.gemms.size(); i++) { tensile_prob.gemms[i].setWorkspaceSize(algo->max_workspace_bytes); @@ -2719,6 +2721,7 @@ rocblaslt_status getBestSolutions(rocblaslt_handle handle, &returnAlgoCount, workspaceBytes, data->problem, + *hardware, fallbackSize); } else if(gemmType == rocblaslt::RocGemmType::ROCBLASLT_GROUPED_GEMM) @@ -2786,6 +2789,7 @@ rocblaslt_status getBestSolutions(rocblaslt_handle handle, &returnAlgoCount, workspaceBytes, data->problem.gemms[0], + *hardware, solutions_fallback.size()); } @@ -2842,6 +2846,10 @@ std::string getSolutionNameFromData(rocblaslt_handle handle, int wgm = 0; int solutionIndex = -1; + std::shared_ptr hardware; + + hardware = Tensile::hip::GetDevice(*deviceProp); + if(gemmType == rocblaslt::RocGemmType::ROCBLASLT_GEMM) { std::shared_ptr data = std::static_pointer_cast(gemmData); @@ -2859,7 +2867,7 @@ std::string getSolutionNameFromData(rocblaslt_handle handle, } if(solutionIndex == -1) return ""; - auto solution = library->getSolutionByIndex(solutionIndex); + auto solution = library->getSolutionByIndex( *hardware, solutionIndex); std::string modifiedString = ""; if(gsu != solution->sizeMapping.globalSplitU && gsu != 0) { @@ -2884,9 +2892,10 @@ std::string getKernelNameFromAlgoIndex(rocblaslt_handle handle, const rocblaslt_ std::shared_ptr deviceProp; auto adapter = get_library_and_adapter(&library, &deviceProp, handle->device); - + std::shared_ptr hardware; + hardware = Tensile::hip::GetDevice(*deviceProp); int* solutionIndex = (int*)algo.data; - auto solution = library->getSolutionByIndex(*solutionIndex); + auto solution = library->getSolutionByIndex(*hardware, *solutionIndex); return solution->kernelName; } @@ -2896,9 +2905,10 @@ std::string getSolutionNameFromAlgoIndex(rocblaslt_handle handle, const rocblasl std::shared_ptr deviceProp; auto adapter = get_library_and_adapter(&library, &deviceProp, handle->device); - + std::shared_ptr hardware; + hardware = Tensile::hip::GetDevice(*deviceProp); int* solutionIndex = (int*)algo.data; - auto solution = library->getSolutionByIndex(*solutionIndex); + auto solution = library->getSolutionByIndex(*hardware , *solutionIndex); return solution->solutionName; } diff --git a/tensilelite/Tensile/Source/client/source/SolutionIterator.cpp b/tensilelite/Tensile/Source/client/source/SolutionIterator.cpp index 1b89ff5788..41cef1d2b6 100644 --- a/tensilelite/Tensile/Source/client/source/SolutionIterator.cpp +++ b/tensilelite/Tensile/Source/client/source/SolutionIterator.cpp @@ -113,7 +113,7 @@ namespace Tensile if(solution.requiredHostWorkspaceSizePerProblem == static_cast(-1)) { solution.requiredHostWorkspaceSizePerProblem - = solution.requiredHostSizeGroupedGemmSingle(problem); + = solution.requiredHostSizeGroupedGemmSingle(problem,m_hardware); } return true; } diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp index 73d4143ca9..4128b71c79 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/ContractionSolution.hpp @@ -263,11 +263,11 @@ namespace Tensile /** * Calculate required workspace size. */ - size_t requiredWorkspaceSize(Problem const& problem) const; - size_t requiredWorkspaceSizeGroupedGemm(std::vector const& problems) const; - size_t requiredHostSizeGroupedGemmSingle(Problem const& problem) const; + size_t requiredWorkspaceSize(Problem const& problem, Hardware const& hardware) const; + size_t requiredWorkspaceSizeGroupedGemm(std::vector const& problems, Hardware const& hardware) const; + size_t requiredHostSizeGroupedGemmSingle(Problem const& problem, Hardware const& hardware) const; - size_t getSKGrid(Hardware const& hardware, size_t tiles) const; + size_t getSKGrid(Problem const& problem, Hardware const& hardware, size_t tiles) const; size_t partialTileSize(size_t skGrid) const; static float computeGranularity(float x); @@ -328,6 +328,7 @@ namespace Tensile virtual std::vector solveGroupedGemmGPU(std::vector const& problems, GroupedInputs const& inputs, + Hardware const& hardware, const void* dUA, const void* workspace, hipStream_t stream) const; @@ -384,6 +385,7 @@ namespace Tensile template KernelInvocation generateSingleCallGroupedGemm(std::vector const& problems, GroupedInputs const& inputs, + Hardware const& hardware, KA& h_args, void const* userArgs = nullptr) const; @@ -418,7 +420,8 @@ namespace Tensile template KernelInvocation generateOutputConversionCallGroupedGemm( - std::vector const& problems, GroupedInputs const& inputs, KA& h_args) const; + std::vector const& problems, GroupedInputs const& inputs, + Hardware const& hardware, KA& h_args) const; template KernelInvocation updateUserArgsOutputConversionCallGroupedGemm( diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/Debug.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/Debug.hpp index 16d8fd108e..4ef10f3890 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/Debug.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/Debug.hpp @@ -79,6 +79,8 @@ namespace Tensile int getSolutionIndex() const; int getGridbasedTopSols() const; + + bool printStreamKGridInfo() const; private: friend LazySingleton; diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/GranularitySelectionLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/GranularitySelectionLibrary.hpp index 9eba1e2361..6af81add46 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/GranularitySelectionLibrary.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/GranularitySelectionLibrary.hpp @@ -240,7 +240,7 @@ namespace Tensile useSolution = true; if(searchType == SolutionLibrarySearchType::DEFAULT) { - size_t ws = (*row.second).requiredWorkspaceSizeGroupedGemm(problems); + size_t ws = (*row.second).requiredWorkspaceSizeGroupedGemm(problems, hardware); for(int idx = 0; idx < problems.size(); idx++) { @@ -263,7 +263,7 @@ namespace Tensile == static_cast(-1)) { (*row.second).requiredHostWorkspaceSizePerProblem - = (*row.second).requiredHostSizeGroupedGemmSingle(problems[0]); + = (*row.second).requiredHostSizeGroupedGemmSingle(problems[0],hardware); } rv.insert(row.second); } diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/MasterSolutionLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/MasterSolutionLibrary.hpp index a427df2bfc..9f258588e7 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/MasterSolutionLibrary.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/MasterSolutionLibrary.hpp @@ -103,12 +103,12 @@ namespace Tensile if(solution->requiredHostWorkspaceSizePerProblem == static_cast(-1)) { solution->requiredHostWorkspaceSizePerProblem - = solution->requiredHostSizeGroupedGemmSingle(problem); + = solution->requiredHostSizeGroupedGemmSingle(problem,hardware); } return solution; } - virtual std::shared_ptr getSolutionByIndex(const int index) const override + virtual std::shared_ptr getSolutionByIndex(Hardware const& hardware, const int index) const override { if(solutions.find(index) == solutions.end()) { @@ -137,7 +137,7 @@ namespace Tensile solution->problemType.groupedGemm, std::numeric_limits::max()); solution->requiredHostWorkspaceSizePerProblem - = solution->requiredHostSizeGroupedGemmSingle(problem); + = solution->requiredHostSizeGroupedGemmSingle(problem,hardware); } return solution; } diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/SingleSolutionLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/SingleSolutionLibrary.hpp index 2c09a30984..2fd8910134 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/SingleSolutionLibrary.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/SingleSolutionLibrary.hpp @@ -136,7 +136,7 @@ namespace Tensile if(!(*solution->hardwarePredicate)(hardware)) return std::shared_ptr(); - size_t ws = (*solution).requiredWorkspaceSizeGroupedGemm(problems); + size_t ws = (*solution).requiredWorkspaceSizeGroupedGemm(problems, hardware); for(int idx = 0; idx < problems.size(); idx++) { @@ -150,7 +150,7 @@ namespace Tensile if(solution->requiredHostWorkspaceSizePerProblem == static_cast(-1)) { solution->requiredHostWorkspaceSizePerProblem - = solution->requiredHostSizeGroupedGemmSingle(problems[0]); + = solution->requiredHostSizeGroupedGemmSingle(problems[0],hardware); } return solution; @@ -220,7 +220,7 @@ namespace Tensile if(searchType == SolutionLibrarySearchType::DEFAULT) { - size_t ws = (*solution).requiredWorkspaceSizeGroupedGemm(problems); + size_t ws = (*solution).requiredWorkspaceSizeGroupedGemm(problems, hardware); for(int idx = 0; idx < problems.size(); idx++) { @@ -266,7 +266,7 @@ namespace Tensile if(solution->requiredHostWorkspaceSizePerProblem == static_cast(-1)) { solution->requiredHostWorkspaceSizePerProblem - = solution->requiredHostSizeGroupedGemmSingle(problems[0]); + = solution->requiredHostSizeGroupedGemmSingle(problems[0],hardware); } return SolutionSet({solution}); } diff --git a/tensilelite/Tensile/Source/lib/include/Tensile/SolutionLibrary.hpp b/tensilelite/Tensile/Source/lib/include/Tensile/SolutionLibrary.hpp index 6d1dd7ace8..17485fd76f 100644 --- a/tensilelite/Tensile/Source/lib/include/Tensile/SolutionLibrary.hpp +++ b/tensilelite/Tensile/Source/lib/include/Tensile/SolutionLibrary.hpp @@ -139,6 +139,12 @@ namespace Tensile return std::shared_ptr(); } + virtual std::shared_ptr getSolutionByIndex(Hardware const& hardware, const int index) const + { + throw std::runtime_error("[getSolutionByIndex] You should not reach here."); + return std::shared_ptr(); + } + virtual std::shared_ptr findBestSolution(MyProblem const& problem, Hardware const& hardware, double* fitness = nullptr) const diff --git a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp index 026308318e..1691d75cab 100644 --- a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp +++ b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -43,6 +44,130 @@ namespace Tensile { + namespace streamk + { + namespace math + { + /** + * Performs `(n + d - 1) / d`, but is robust against the case where + * `(n + d - 1)` would overflow. + */ + template + __device__ __host__ inline constexpr N safe_ceil_div(N n, D d) + { + // Static cast to undo integral promotion. + return static_cast(n / d + (n % d != 0 ? 1 : 0)); + } + } // namespace math + + constexpr size_t num_iters_per_cta( + size_t BLK_M, size_t BLK_N, size_t BLK_K, size_t m, size_t n, size_t k, int g) + { + return math::safe_ceil_div(math::safe_ceil_div(m, BLK_M) * math::safe_ceil_div(n, BLK_N) + * math::safe_ceil_div(k, BLK_K), + g); + } + + constexpr size_t number_of_output_tiles(size_t BLK_M, size_t BLK_N, size_t m, size_t n) + { + size_t m_tiles = math::safe_ceil_div(m, BLK_M); + size_t n_tiles = math::safe_ceil_div(n, BLK_N); + return m_tiles * n_tiles; + } + + constexpr size_t num_fixup_peers(size_t BLK_K, size_t k, size_t iters_per_cta) + { + return math::safe_ceil_div(math::safe_ceil_div(k, BLK_K), iters_per_cta); + } + + std::tuple predicted_runtime(size_t BLK_M, + size_t BLK_N, + size_t BLK_K, + size_t m, + size_t n, + size_t k, + int g, + double a, + double b, + double c, + double d) + { + size_t iters_per_cta = num_iters_per_cta(BLK_M, BLK_N, BLK_K, m, n, k, g); + size_t fixup_peers = num_fixup_peers(BLK_K, k, iters_per_cta); + + return {a + (b * (fixup_peers > 1)) + (c * iters_per_cta) + (d * (fixup_peers - 1)), + iters_per_cta, + fixup_peers}; + } + + int best_predicted_grid_size(size_t BLK_M, + size_t BLK_N, + size_t BLK_K, + size_t m, + size_t n, + size_t k, + int grid_start = 1, + int grid_end = 304) + { + static const bool debug = Debug::Instance().printStreamKGridInfo(); + + // Fixed overhead alpha (a), fixed-size cost incurred by + // each work-group, e.g. the grid launch latency, the initial + // compulsary cache misses, the cost of writing the final output tile + // to C. + double a = 5.04 + 8.30; + + // Beta (b) incorporates conditional costs of outputting temporary partial + // sums for scenarios where the number of output tiles does not quantize + // perfectly across the number of processors. + double b = 5.47; + + // c represents instruction and stall workload of each MAC-iteration. + double c = 4.17; + + // Delta (d) is the cost of reading and accumulating the partial sums from + // other work-groups covering the same tile. + double d = 18.59; + + // std::vector runtimes; + std::pair min_grid_runtime; + min_grid_runtime.second = std::numeric_limits::max(); + int g = grid_start; + + // Predict the number of CTAs to use between 1 and 304 + for(; g <= grid_end; ++g) + { + auto [runtime, iters_per_cta, fixup_peers] + = predicted_runtime(BLK_M, BLK_N, BLK_K, m, n, k, g, a, b, c, d); + + if(debug) + { + std::cout << "grid size: " << g << ", runtime: " << runtime + << ", iters_per_cta: " << iters_per_cta + << ", fixup_peers: " << fixup_peers << ", m: " << m << ", n: " << n + << ", k: " << k << ", a: " << a << ", b: " << b << ", c: " << c + << ", d: " << d << std::endl; + } + + if(min_grid_runtime.second > runtime) + { + min_grid_runtime.first = g; + min_grid_runtime.second = runtime; + } + } + + if(debug) + { + std::cout << "Number of Output Tiles: " + << number_of_output_tiles(BLK_M, BLK_N, m, n) << std::endl; + std::cout << "Minimum runtime: " << min_grid_runtime.second + << " @ grid size: " << min_grid_runtime.first << std::endl; + } + + return min_grid_runtime.first; + } + } // namespace streamk + enum class KERNELARGTYPE { NORMAL = 0, @@ -599,7 +724,7 @@ namespace Tensile size_t cuCount = 0; auto tiles = problem.getNumTiles(sizeMapping); - size_t skGrid = getSKGrid(*hardware, tiles); + size_t skGrid = getSKGrid(problem, *hardware, tiles); // StreamK workspace + flags args.template append("ws", inputs.ws); void* ws = inputs.ws; @@ -886,7 +1011,7 @@ namespace Tensile cuCount = pAMDGPU->computeUnitCount; if(sizeMapping.streamK != 0) { - skGrid = getSKGrid(hardware, tiles); + skGrid = getSKGrid(problem, hardware, tiles); // TODO enable grid size change once code gen changes are implemented // rv.numWorkGroups.x = skGrid; // rv.numWorkGroups.y = 1; @@ -1061,6 +1186,7 @@ namespace Tensile KernelInvocation ContractionSolution::generateSingleCallGroupedGemm( std::vector const& problems, ContractionSolution::GroupedInputs const& inputs, + Hardware const& hardware, KA& h_args, void const* userArgs) const { @@ -1104,7 +1230,7 @@ namespace Tensile } if constexpr(std::is_same::value) - workspaceOffsetInByte += requiredWorkspaceSize(problem); + workspaceOffsetInByte += requiredWorkspaceSize(problem, hardware); } } @@ -1665,6 +1791,7 @@ namespace Tensile KernelInvocation ContractionSolution::generateOutputConversionCallGroupedGemm( std::vector const& problems, ContractionSolution::GroupedInputs const& inputs, + Hardware const& hardware, KA& h_args) const { KernelInvocation rv; @@ -1708,7 +1835,7 @@ namespace Tensile outputConversionCallArgs( problem, inputs.grouped[idx], workspaceOffsetInByte, h_args); if constexpr(std::is_same::value) - workspaceOffsetInByte += requiredWorkspaceSize(problem); + workspaceOffsetInByte += requiredWorkspaceSize(problem, hardware); } if constexpr(std::is_same::value) @@ -2392,18 +2519,18 @@ namespace Tensile // } if(debug) - rv.push_back(generateSingleCallGroupedGemm(problems, inputs, h_args)); + rv.push_back(generateSingleCallGroupedGemm(problems, inputs, hardware, h_args)); else - rv.push_back(generateSingleCallGroupedGemm(problems, inputs, h_args)); + rv.push_back(generateSingleCallGroupedGemm(problems, inputs, hardware, h_args)); if(sizeMapping.globalAccumulation == 2 && gsu > 1) { if(debug) rv.push_back( - generateOutputConversionCallGroupedGemm(problems, inputs, h_args)); + generateOutputConversionCallGroupedGemm(problems, inputs, hardware, h_args)); else rv.push_back( - generateOutputConversionCallGroupedGemm(problems, inputs, h_args)); + generateOutputConversionCallGroupedGemm(problems, inputs, hardware, h_args)); } if(debug) @@ -2431,6 +2558,7 @@ namespace Tensile std::vector ContractionSolution::solveGroupedGemmGPU(std::vector const& problems, GroupedInputs const& inputs, + Hardware const& hardware, const void* dUA, const void* workspace, hipStream_t stream) const @@ -2446,9 +2574,9 @@ namespace Tensile // Here we only update the pointer int h_args = 1; // Dummy if(debug) - rv.push_back(generateSingleCallGroupedGemm(problems, inputs, h_args, dUA)); + rv.push_back(generateSingleCallGroupedGemm(problems, inputs, hardware, h_args, dUA)); else - rv.push_back(generateSingleCallGroupedGemm(problems, inputs, h_args, dUA)); + rv.push_back(generateSingleCallGroupedGemm(problems, inputs, hardware, h_args, dUA)); auto gsu = problems[0].getParams().gsu() > 0 ? problems[0].getParams().gsu() : sizeMapping.globalSplitU; @@ -2494,7 +2622,7 @@ namespace Tensile throw std::runtime_error("Unsupported Device memory type."); } - return solveGroupedGemmGPU(problems, inputs, *dUA, inputs.ws, stream); + return solveGroupedGemmGPU(problems, inputs,hardware, *dUA, inputs.ws, stream); } void ContractionSolution::relaseDeviceUserArgs(void* dUA, void* dUAHost) @@ -2617,54 +2745,71 @@ namespace Tensile return pass; } - size_t ContractionSolution::requiredWorkspaceSize(Problem const& problem) const + size_t ContractionSolution::requiredWorkspaceSize(Problem const& problem, + Hardware const& hardware) const { - // TODO: Pass GSU from problem and change value[2] to gsu if gsu != default value size_t size = 0; - size_t gsu - = problem.getParams().gsu() > 0 ? problem.getParams().gsu() : sizeMapping.globalSplitU; - size_t gsuMultiplier = gsu > 1 ? gsu : 0; - size += problem.d().totalLogicalElements() * sizeMapping.workspaceSizePerElemC - * gsuMultiplier; - if(problemType.useGradient && problemType.useBias - && problem.getParams().biasEnum() != DataType::None) + if(sizeMapping.streamK > 0 && sizeMapping.streamKAtomic == 0) { - if(problem.biasSrc() == ContractionProblemGemm::TENSOR::A) - { - size += problem.freeSizeA(0) * sizeMapping.workspaceSizePerElemBias * gsuMultiplier; - } - else if(problem.biasSrc() == ContractionProblemGemm::TENSOR::B) + auto tiles = problem.getNumTiles(sizeMapping); + size_t skGrid = getSKGrid(problem, hardware, tiles); + // Get space required for partial tiles + size += partialTileSize(skGrid); + // Add space for flags + // Flags for partial tiles - dword per flag for fast addressing and comparisons + // If tiles is evenly divided by grid size flags are not needed (DP mode) + if(tiles % skGrid != 0) + size += skGrid * 4; + // size *= batches; // TODO need tile and flag per batch + } + else + { + // TODO: Pass GSU from problem and change value[2] to gsu if gsu != default value + size_t gsu + = problem.getParams().gsu() > 0 ? problem.getParams().gsu() : sizeMapping.globalSplitU; + size_t gsuMultiplier = gsu > 1 ? gsu : 0; + + size += problem.d().totalLogicalElements() * sizeMapping.workspaceSizePerElemC + * gsuMultiplier; + if(problemType.useGradient && problemType.useBias + && problem.getParams().biasEnum() != DataType::None) { - size += problem.freeSizeB(0) * sizeMapping.workspaceSizePerElemBias * gsuMultiplier; + if(problem.biasSrc() == ContractionProblemGemm::TENSOR::A) + { + size += problem.freeSizeA(0) * sizeMapping.workspaceSizePerElemBias * gsuMultiplier; + } + else if(problem.biasSrc() == ContractionProblemGemm::TENSOR::B) + { + size += problem.freeSizeB(0) * sizeMapping.workspaceSizePerElemBias * gsuMultiplier; + } + else if(problem.biasSrc() == ContractionProblemGemm::TENSOR::D && (gsuMultiplier == 0)) + { + size += problem.d().totalLogicalElements() * sizeMapping.workspaceSizePerElemBias + * gsu; + } } - else if(problem.biasSrc() == ContractionProblemGemm::TENSOR::D && (gsuMultiplier == 0)) + + // Custom kernel synchronizer + if(gsu > 1 && sizeMapping.globalAccumulation == 3) { - size += problem.d().totalLogicalElements() * sizeMapping.workspaceSizePerElemBias - * gsu; + size += (int)ceil(problem.d().sizes()[0] / (float)sizeMapping.macroTile.x) + * (int)ceil(problem.d().sizes()[1] / (float)sizeMapping.macroTile.y) + * sizeMapping.waveNum * sizeof(int32_t); } } - - // Custom kernel synchronizer - if(gsu > 1 && sizeMapping.globalAccumulation == 3) - { - size += (int)ceil(problem.d().sizes()[0] / (float)sizeMapping.macroTile.x) - * (int)ceil(problem.d().sizes()[1] / (float)sizeMapping.macroTile.y) - * sizeMapping.waveNum * sizeof(int32_t); - } - return size; } size_t ContractionSolution::requiredWorkspaceSizeGroupedGemm( - std::vector const& problems) const + std::vector const& problems, Hardware const& hardware) const { size_t sizeInByte = 0; for(int i = 0; i < problems.size(); i++) { auto problem = problems[i]; - sizeInByte += requiredWorkspaceSize(problem); + sizeInByte += requiredWorkspaceSize(problem, hardware); } ContractionGroupedInputs inputs; for(int i = 0; i < problems.size(); i++) @@ -2673,14 +2818,14 @@ namespace Tensile inputs.grouped.push_back(unit); } auto h_args = KernelArgumentsCounter(); - generateSingleCallGroupedGemm(problems, inputs, h_args); + generateSingleCallGroupedGemm(problems, inputs, hardware, h_args); if(sizeMapping.globalAccumulation) - generateOutputConversionCallGroupedGemm(problems, inputs, h_args); + generateOutputConversionCallGroupedGemm(problems, inputs, hardware, h_args); sizeInByte += h_args.size(); return sizeInByte; } - size_t ContractionSolution::requiredHostSizeGroupedGemmSingle(Problem const& problem) const + size_t ContractionSolution::requiredHostSizeGroupedGemmSingle(Problem const& problem, Hardware const& hardware) const { if(!problemType.groupedGemm) return 0; @@ -2694,42 +2839,106 @@ namespace Tensile inputs.grouped.push_back(unit); } auto h_args = KernelArgumentsCounter(); - generateSingleCallGroupedGemm(tmpProblem, inputs, h_args); + generateSingleCallGroupedGemm(tmpProblem, inputs, hardware, h_args); if(sizeMapping.globalAccumulation) - generateOutputConversionCallGroupedGemm(tmpProblem, inputs, h_args); + generateOutputConversionCallGroupedGemm(tmpProblem, inputs, hardware, h_args); return h_args.size(); } - size_t ContractionSolution::getSKGrid(Hardware const& hardware, size_t tiles) const + size_t ContractionSolution::getSKGrid(Problem const& problem, Hardware const& hardware, size_t tiles) const { AMDGPU const* pAMDGPU = dynamic_cast(&hardware); - if(pAMDGPU->skFixedGrid > 0) - return pAMDGPU->skFixedGrid; + assert(pAMDGPU != nullptr && pAMDGPU->computeUnitCount != 0); size_t cuCount = pAMDGPU->computeUnitCount; - size_t skGrid = cuCount; - if(pAMDGPU->skDynamicGrid == 2 && tiles > skGrid) + + // User-specified grid size for Stream-K kernel. + if(pAMDGPU->skFixedGrid > 0) + { + return pAMDGPU->skFixedGrid; + } + + // Dynamically pick the minimum between the cuCount or number of tiles. + else if(pAMDGPU->skDynamicGrid == 1) + { + return min(cuCount, tiles); + } + + // Dynamically pick the minimum between the cuCount or number of tiles, + // and scale down really large sizes to use fewer CUs for power/energy savings. + else if(pAMDGPU->skDynamicGrid == 2) { - for(size_t i = 1; i <= 32; i *= 2) + size_t skGrid = cuCount; + if(tiles > skGrid) { - size_t tilesPerCU = CeilDivide(i * tiles, skGrid); - size_t reducedGrid = CeilDivide(i * tiles, tilesPerCU); - float utilization = ((float)reducedGrid) / ((float)skGrid); - if(utilization > 0.75f) + for(size_t i = 1; i <= 32; i *= 2) { - if(utilization < 1.0f) - skGrid = reducedGrid; - break; + size_t tilesPerCU = CeilDivide(i * tiles, cuCount); + size_t reducedGrid = CeilDivide(i * tiles, tilesPerCU); + float utilization = ((float)reducedGrid) / ((float)cuCount); + if(utilization > 0.75f) + { + if(utilization < 1.0f) + skGrid = reducedGrid; + break; + } } } + + return min(skGrid, tiles); + } + + // Dynamically predict the best grid-size by weighing the cost of the fix-up + // step and the cost of processing MAC-loop instructions. When the cost of fix-up + // is the bottleneck, use smaller grid size. + // Architecture dependent. + else if(pAMDGPU->skDynamicGrid == 3) + { + size_t x = 1; + size_t y = 1; + size_t z = 1; + for(size_t i = 0; i < problem.freeIndicesA().size(); i++) + { + x *= problem.freeSizeA(i); + } + for(size_t i = 0; i < problem.freeIndicesB().size(); i++) + { + y *= problem.freeSizeB(i); + } + // TODO Batch dimension + for(size_t i = 0; i < problem.boundIndices().size(); ++i) + { + z *= problem.boundSize(i); + } + + return streamk::best_predicted_grid_size(sizeMapping.macroTile.x, + sizeMapping.macroTile.y, + sizeMapping.depthU, + x, + y, + z, + cuCount); + } + + // Limit the CUs Stream-K is launched on either max or the specified, + // whichever is minimum. + else if(pAMDGPU->skMaxCUs > 0) + { + return min(cuCount, pAMDGPU->skMaxCUs); + } + + // Multiply the cuCount with a constant factor (c), and launch + // c * cuCount number of workgroups for Stream-K. + else if(pAMDGPU->skGridMultiplier > 1) + { + return cuCount * pAMDGPU->skGridMultiplier; + } + + // If no option is specified, launch exactly cuCount worth of workgroups. + else + { + return cuCount; } - if(pAMDGPU->skMaxCUs > 0) - skGrid = min(skGrid, pAMDGPU->skMaxCUs); - if(pAMDGPU->skDynamicGrid) - skGrid = min(skGrid, tiles); - if(pAMDGPU->skGridMultiplier > 1) - skGrid = skGrid * pAMDGPU->skGridMultiplier; - return skGrid; } size_t ContractionSolution::partialTileSize(size_t skGrid) const diff --git a/tensilelite/Tensile/Source/lib/source/Debug.cpp b/tensilelite/Tensile/Source/lib/source/Debug.cpp index 9bf6d70d08..96af0bb2f1 100644 --- a/tensilelite/Tensile/Source/lib/source/Debug.cpp +++ b/tensilelite/Tensile/Source/lib/source/Debug.cpp @@ -145,6 +145,11 @@ namespace Tensile return m_gridbasedTopSols; } + bool Debug::printStreamKGridInfo() const + { + return m_value & 0x80000; + } + Debug::Debug() : m_value(DEBUG_SM) , m_value2(DEBUG_SM2) From 5ce3017e4628ad1858018f956ea78472fcd69fe9 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Mon, 22 Jul 2024 20:52:23 +0000 Subject: [PATCH 16/36] Fixup/partials, flag init, bug fixes --- tensilelite/Tensile/Common.py | 2 + tensilelite/Tensile/Components/StreamK.py | 776 +++++++++++++++++++- tensilelite/Tensile/KernelWriter.py | 1 + tensilelite/Tensile/KernelWriterAssembly.py | 40 +- tensilelite/Tensile/KernelWriterModules.py | 26 +- 5 files changed, 794 insertions(+), 51 deletions(-) diff --git a/tensilelite/Tensile/Common.py b/tensilelite/Tensile/Common.py index 88568fb832..1c54c9a38c 100644 --- a/tensilelite/Tensile/Common.py +++ b/tensilelite/Tensile/Common.py @@ -1017,6 +1017,7 @@ def getArchitectureName(gfxName): "NonTemporalC": list(range(0,8)), "NonTemporalA": list(range(0,8)), "NonTemporalB": list(range(0,8)), + "NonTemporalWS": list(range(0,8)), "NonTemporalMetadata": list(range(0,8)), "NonTemporal": list(range(-1,8)), @@ -1153,6 +1154,7 @@ def getArchitectureName(gfxName): {"NonTemporalC": [ 0 ] }, {"NonTemporalA": [ 0 ] }, {"NonTemporalB": [ 0 ] }, + {"NonTemporalWS": [ 0 ] }, {"NonTemporalMetadata": [ 0 ] }, {"NonTemporal": [ -1 ] }, {"PreloadKernArgs": [ True ] }, diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index 24189b6994..e5d4474f45 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -23,7 +23,8 @@ from ..TensileInstructions import Module, Label, SAddU32, RegisterPoolResource, sgpr, scalarStaticDivideAndRemainder, \ SCmpLtU32, SCSelectB32, sMagicDivAlg2, SMulI32, SSubU32, SMinU32, SMovB32, SCBranchSCC1, SCmpLeU32, VMovB32, vgpr, \ SAddCU32, SCmpGtU32, SCMovB32, SAddI32, SCmpEQU32, SCBranchSCC0, SLShiftLeftB32, SLoadB32, SWaitCnt, SMEMModifiers, \ - log2, SBarrier, SStoreB32, SLongBranchPositive, SBranch, ceilDivide, replaceHolder, SNop, staticMultiply + log2, SBarrier, SStoreB32, SLongBranchPositive, SBranch, ceilDivide, replaceHolder, SNop, staticMultiply, SSleep, \ + VAddF32 from ..Common import print2 # from ..TensileInstructions.Containers import SMEMModifiers from ..Component import Component @@ -249,10 +250,10 @@ def calculateLoopNumIterCommon(self, writer, kernel, loopCounterName, loopIdx, t return module @abc.abstractmethod - def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements): + def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct): pass - def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, elements): + def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct): module = Module("StreamK Common storeBranches") # No branches for atomic mode @@ -296,11 +297,12 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele module.add(SCmpEQU32(src0=sgpr(tmpSgpr+2), src1=1, comment="check if ready")) module.add(SCBranchSCC0(labelName=skFixupLabel.getLabelName(), comment="if flag not set, wait and check again")) + module.add(SMovB32(dst=sgpr(tmpSgpr+2), src=0, comment="flag data")) + module.add(SStoreB32(src=sgpr(tmpSgpr+2), base=sgpr("AddressFlags", 2), soffset=sgpr(tmpSgpr), smem=SMEMModifiers(glc=1), comment="reset flag")) writer.sgprPool.checkIn(tmpSgpr) - # TODO FIXUP STEP!!!!!!!!!!!!!!! - fixupEdge = [False] # Temporary hack to test no edge variant - # kStr += self.fixupStep(kernel, vectorWidths, elements, fixupEdge, tmpVgpr, tmpCVTVgpr, sCtaIdx, skStoreLabel) + fixupEdge = [False] # Test no edge variant + module.add(self.fixupStep(writer, kernel, vectorWidths, elements, fixupEdge, tmpVgpr, cvtVgprStruct, sCtaIdx)) if kernel["StreamK"] >= 2: sIterCount = writer.sgprPool.checkOut(1, "iterCount", preventOverflow=0) @@ -693,19 +695,16 @@ def partialsWriteBatch(self, writer, kernel, ss, batchIdx, applyAlpha, beta, edg # TODO KUPO!!!!!!!!!!!!!!!! # newSumIdxV = sumIdxV - writer.states.c.startVgprValu # covers sgemm, gemm_ex(HHS/HSS/BBS/BSS (HPA=T)), int8 (int8x4?) - if kernel["ProblemType"]["ComputeDataType"].isInt32() or \ - kernel["ProblemType"]["ComputeDataType"].isSingle(): # covers sgemm/gemm_ex(HHS/HSS/BBS/BSS) - if writer.db["ForceExpectedValue"]: - module.add(VMovB32(dst=vgpr("ValuC+%u"%sumIdxV), src=writer.db["ValueCExpectedValue"], comment="force expected value")) - # module.add(VMovB32(dst=vgpr("ValuC+%u"%newSumIdxV), src=self.debugConfig["ValueCExpectedValue"], comment="force expected value" )) - if writer.db["ForceVSerial"]: - module.add(VMovB32(dst=vgpr("ValuC+%u"%sumIdxV), src=vgpr("Serial"), comment="force expected value to serial")) - # module.add(VMovB32(dst=vgpr("ValuC+%u"%newSumIdxV), src=vgpr("Serial"), comment="force expected value to serial" )) - if writer.db["CheckValueC"]: - module.add(SMovB32(dst=sgpr(tmpS01), src=writer.db["ValueCExpectedValue"], comment="Move expected value")) - module.add(writer.getCmpAssert(writer.asmAssert.eq, vgpr("ValuC+%u"%sumIdxV), sgpr(tmpS01))) - # module.add(SMovB32(dst=sgpr(self.tmpS01), src=self.debugConfig["ValueCExpectedValue"], comment="Move expected value")) - # module.add(self.parentWriter.getCmpAssert(self.parentWriter.asmAssert.eq, vgpr("ValuC+%u"%newSumIdxV), sgpr(self.tmpS01))) + if kernel["ProblemType"]["ComputeDataType"].isInt32() or kernel["ProblemType"]["ComputeDataType"].isSingle(): + if writer.db["ForceExpectedValue"]: + module.add(VMovB32(dst=vgpr("ValuC+%u"%sumIdxV), src=writer.db["ValueCExpectedValue"], comment="force expected value")) + # module.add(VMovB32(dst=vgpr("ValuC+%u"%newSumIdxV), src=self.debugConfig["ValueCExpectedValue"], comment="force expected value" )) + if writer.db["ForceVSerial"]: + module.add(VMovB32(dst=vgpr("ValuC+%u"%sumIdxV), src=vgpr("Serial"), comment="force expected value to serial")) + # module.add(VMovB32(dst=vgpr("ValuC+%u"%newSumIdxV), src=vgpr("Serial"), comment="force expected value to serial" )) + if writer.db["CheckValueC"]: + module.add(SMovB32(dst=sgpr(tmpS01), src=writer.db["ValueCExpectedValue"], comment="Move expected value")) + module.add(writer.getCmpAssert(writer.asmAssert.eq, vgpr("ValuC+%u"%sumIdxV), sgpr(tmpS01))) module.addComment1("apply mask, calc new C and issue writes") #kStr += self.bomb() # can see store addresses just before the store inst @@ -823,6 +822,731 @@ def partialsWriteBatch(self, writer, kernel, ss, batchIdx, applyAlpha, beta, edg return module + def fixupStep(self, writer, kernel, vectorWidths, elements, edges, tmpVgpr, cvtVgprStruct, sCtaIdx): + module = Module("StreamK Common fixupStep") + + fixupLabels = {} + for edge in edges: + fixupLabels[edge] = Label(writer.labels.getNameInc("Fixup_E%u" % ( 1 if edge else 0)), comment="") + + # branch if Edge0 or Edge1 + if False in edges and True in edges: + module.add(writer.checkIsEdge(kernel, tmpSgprInfo, fixupLabels[True], fixupLabels[True])) + + # by now we either jumped to E1 or stayed at E0 + for edge in edges: + # write label for batch case + module.add(fixupLabels[edge]) + + # PreLoopVmcntCaseStr = "" + # # not generate Case 2 if StoreCInUnroll with StoreVectorWidth==1 (Case 2 will be same as Case 3) + # if self.canOptimizePreLoopLWVmcnt: + # if edge or (kernel["StoreCInUnroll"] and kernel["StoreVectorWidth"]==1): + # self.currPreLoopVmcntCase = PreLoopVmcntCase.OrdNLL_E1_Store + # else: + # self.currPreLoopVmcntCase = PreLoopVmcntCase.OptNLL_Store + # PreLoopVmcntCaseStr = inst("s_mov_b32", sgpr("PreLoopLWVmcntCase"), hex(self.currPreLoopVmcntCase.value), \ + # "for optimizing next PreLoop LW vmcnt, set to Case%u"%self.currPreLoopVmcntCase.value) + # # reset vmcnt if the dict has this key (OptNLL_Store, OrdNLL_E1_Store), + # # OrdNLL_B1_Store is excluded + # if self.currPreLoopVmcntCase in self.preLoopVmcntDict: + # self.preLoopVmcntDict[self.currPreLoopVmcntCase] = 0 + + edgeI = edge + #edgeI = True # set to True to disable vector stores + gwvw = vectorWidths[edgeI] + + ######################################## + # Calculate Vgprs for Write Batching + ######################################## + + # ss = StoreState(writer, kernel, gwvw, edge, beta, False, elements[edgeI], isWorkspace=True) + ss = StoreState(writer, kernel, gwvw, edge, True, False, elements[edgeI], isWorkspace=True) + # ss = StoreState(writer, kernel, gwvw, edge, False, False, elements[edgeI], isWorkspace=True) + + # how many vgprs are needed for zero elements + # 2 for addressC in vgpr for addition - already checked out + # 2 for coord0,1 of thread - already checked out + # 2 for tmp - already checked out + + # 5 = how many vgprs are needed per element (flat) + # - 2 for addr + # - 3 for GLOBAL_OFFSET_C calculation (can overlap below, therefore max) + # - if beta gwvw*rpe for new value + # - if atomic 2*rpe for old and cmp values + + # print("numVgprsPerAddr=%u, numVgprsPerDataPerVI=%u, numVgprPerValuC=%u"%(self.ss.cfg.numVgprsPerAddr, self.ss.cfg.numVgprsPerDataPerVI, self.ss.cfg.numVgprPerValuC)) + # numVgprsPerElement = self.ss.cfg.numVgprPerValuC*gwvw + self.ss.cfg.numVgprsPerAddr + int(ceil(self.ss.cfg.numVgprsPerDataPerVI * gwvw)) + + # if kernel["GroupLoadStore"] and kernel["ProblemType"]["UseBeta"]: + # numVgprsPerElement += self.ss.cfg.numVgprsPerAddr + + #print self.vgprPool.state() + # Use VGPR up to next occupancy threshold: + maxVgprs = writer.getMaxRegsForOccupancy(kernel["NumThreads"], writer.vgprPool.size(), \ + writer.getLdsSize(kernel), writer.agprPool.size(), writer.states.doubleVgpr) + if writer.states.serializedStore: # get aggressive when serializedStore is on; not necessarily exclusive to this parameter + # len(elements[edgeI]) + # tl = [] + # for i in range(self.vgprPool.size()-self.vgprPool.available(), maxVgprs): + # tl.append(self.vgprPool.checkOut(1, "grow-pool up to next occupancy for GlobalWrite")) + # for t in tl: + # self.vgprPool.checkIn(t) + writer.vgprPool.growPool(writer.vgprPool.size()-writer.vgprPool.available(), maxVgprs, 1, \ + "grow-pool up to next occupancy for GlobalWrite") + # align = 1 + # # align adjustment + # if self.ss.cfg.numVgprsPerAddr > 1: + # align = max(align, self.ss.cfg.numVgprsPerAddr) + # if self.ss.cfg.numVgprPerValuC*gwvw > 1: + # align = max(align, self.ss.cfg.numVgprPerValuC*gwvw) + # if int(ceil(self.ss.cfg.numVgprsPerDataPerVI * gwvw)) > 1: + # align = max(align, int(ceil(self.ss.cfg.numVgprsPerDataPerVI * gwvw))) + numVgprAvailable = writer.vgprPool.availableBlock(ss.numVgprsPerElement, ss.align) + + # Grow the register pool if needed - we need enough regs for at least one element + # Unfortunate since this means the write logic is setting the VGPR requirement + # for the entire kernel but at least we have a functional kernel. + # Before growing the pool, see if we can shrink the write vector width instead? + # TODO : the vgprSerial is needed for-ever and if we grow here will split the + # range of the tmps. Maybe want to move vgprSerial to first vgpr? + + # TODO: Minimum elems for StoreRemap + # TODO: Which of DataType or DestDataType is in a better sense? 0114: Check Using DestDataType + HSS + minElements = 1 + if kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16(): + minElements = 2 + elif kernel["ProblemType"]["DataType"].is8bitFloat(): + minElements = 4 + minNeeded = minElements * ss.numVgprsPerElement + + shrinkDb = 0 + if shrinkDb: + print("numVgprAvailable=", numVgprAvailable, "minElements=", minElements, "minNeeded=", minNeeded) + + if numVgprAvailable < minNeeded: + gwvwOrig = gwvw + currentOccupancy = writer.getOccupancy(kernel["NumThreads"], writer.getLdsSize(kernel), \ + writer.vgprPool.size(), writer.agprPool.size(), writer.states.doubleVgpr) + futureOccupancy = writer.getOccupancy(kernel["NumThreads"], writer.getLdsSize(kernel), \ + writer.vgprPool.size() - numVgprAvailable + minNeeded, writer.agprPool.size(), writer.states.doubleVgpr) + + if shrinkDb: + print("currentOccupancy=%u futureOccupancy=%u VGPRs=%u numVgprAvail=%u vgprPerElem=%u" \ + % (currentOccupancy, futureOccupancy, writer.vgprPool.size(), \ + numVgprAvailable, minElements*ss.numVgprsPerElement)) + if futureOccupancy > currentOccupancy: + if shrinkDb: + print("warning: %s growing VGPR for GlobalWrite batching - this may bloat VGPR usage" % \ + (writer.states.kernelName)) + print(" numVgprAvailable=", numVgprAvailable, \ + "numVgprsPerElement=", ss.numVgprsPerElement, \ + "gwvw=", gwvw) + elif gwvw != gwvwOrig: + ss.gwvw = gwvw # make both representations consistent + if shrinkDb: + print2(3, "info: %s shrank gwvw from %u to %u but kept occupancy same=%u." \ + % (writer.states.kernelName, gwvwOrig, gwvw, currentOccupancy)) + + if numVgprAvailable < minElements*ss.numVgprsPerElement: + print2(3, "info: growing pool += %d * %d for GlobalWrite\n" \ + % (minElements,ss.numVgprsPerElement)) + print2(3, writer.vgprPool.state()) + # tl = [] + # for i in range(0,minElements): + # tl.append(self.vgprPool.checkOut(numVgprsPerElement, "grow-pool for GlobalWrite")) + # for t in tl: + # self.vgprPool.checkIn(t) + writer.vgprPool.growPool(0, minElements, ss.numVgprsPerElement, \ + "grow-pool for GlobalWrite") + numVgprAvailable = writer.vgprPool.available() + print2(3, writer.vgprPool.state()) + + # print("NumVgprAvailable", numVgprAvailable) + if ss.numVgprsPerElement: + numElementsPerBatch = numVgprAvailable // ss.numVgprsPerElement + else: + numElementsPerBatch = len(elements[edgeI]) # max, do 'em all + + # assert(self.numVgprValuC % gwvw == 0) # sanity check + + numElementsPerBatch = numElementsPerBatch if not kernel["NumElementsPerBatchStore"] else min(kernel["NumElementsPerBatchStore"],numElementsPerBatch) + + if shrinkDb: + print("NumElementsPerBatch=", numElementsPerBatch, "LimitedBySgprs=", ss.cfg.numElementsPerBatchLimitedBySgprs, \ + "WARNING" if ss.cfg.numElementsPerBatchLimitedBySgprs < numElementsPerBatch else "okay") + if ss.cfg.numElementsPerBatchLimitedBySgprs < numElementsPerBatch: + numElementsPerBatch = ss.cfg.numElementsPerBatchLimitedBySgprs + + # TODO: Which of DataType or DestDataType is in a better sense? 0114: Check Using DestDataType + HSS + if (kernel["ProblemType"]["DataType"].isHalf() or kernel["ProblemType"]["DataType"].isBFloat16()): + # only do an even number of halves - since these share hi/lo pieces of some registers? + if numElementsPerBatch > 1: + numElementsPerBatch = int(numElementsPerBatch/2)*2 + elif not kernel["EnableMatrixInstruction"]: + # (excluding MFMA+LSU case. It can work without an issue) + # The globalWriteBatch routine below can't handle odd elements per batch + # and 0 elements per batch is illegal. + # so if we don't have *GPR resources to handle a larger batch then need + # to mark overflowedResources rather than generate a kernel that won't work. + # It might be possible to fix globalWriteBatch to handle this case but these + # are likely to be low-performing so likely not worth optimizing. + if shrinkDb: + print("WARNING: half requires at least two elements per batch") + self.overflowedResources = 3 + #elif kernel["ProblemType"]["DataType"].is8bitFloat(): + # if numElementsPerBatch > 1: + # numElementsPerBatch = int(numElementsPerBatch/4)*4 + + assert numElementsPerBatch > 0, "numElementsPerBatch=0 for %s"%writer.states.kernelName + + # if no atomics and no edge, then write whole vectors + # ERROR commented out in globalWriteELements, causes numVectorsPerBatch to not be int + # if not edge: # not atomic and + # numVectorsPerBatch = numElementsPerBatch / kernel["GlobalWriteVectorWidth"] + # #print " NumVectorsPerBatch", numVectorsPerBatch + # numElementsPerBatch = numVectorsPerBatch * kernel["GlobalWriteVectorWidth"] + numBatches = max(1, ceilDivide(len(elements[edgeI]),numElementsPerBatch)) + + numSgprs = ss.cfg.numTempSgprPerBatch + ss.cfg.numMaskSgprPerBatch + ss.cfg.numMaskSgprPerElement * numElementsPerBatch + + if writer.db["PrintStoreRegisterDb"]: + print("edgeI", edgeI, "NumBatches", numBatches, "NumElementsPerBatch", numElementsPerBatch, "numVgprsPerElement", ss.numVgprsPerElement, "len(elements[edgeI])", len(elements[edgeI])) + print ("numSgprs=", numSgprs, "sgprPool.size()=", writer.sgprPool.size(), "numTempSgprPerBatch=", ss.cfg.numTempSgprPerBatch, + "numMaskSgprPerBatch=", ss.cfg.numMaskSgprPerBatch, "numMaskSgprPerElement=", ss.cfg.numMaskSgprPerElement) + print(writer.sgprPool.state()) + module.addComment1("edge=%d, allocate %u sgpr. perBatchTmpS=%u perBatchMaskS=%u perElementMaskS=%u elementsPerBatch=%u" % + (edgeI, numSgprs, ss.cfg.numTempSgprPerBatch, ss.cfg.numMaskSgprPerBatch, ss.cfg.numMaskSgprPerElement, numElementsPerBatch)) + #kStr += "// storeStats, %d, %d, %d\n"% (edgeI, numSgprs, numElementsPerBatch) + # so if we don't have *GPR resources to handle a larger batch then need + # to mark overflowedResources rather than generate a kernel that won't work. + + with writer.allocTmpSgpr(numSgprs, 2) as tmpSgprRes: + tmpSgpr = tmpSgprRes.idx + elementSgprs = tmpSgpr + ss.cfg.numTempSgprPerBatch + + codeAccVgprRead = deepcopy(writer.codes.accVgprRead) if writer.states.serializedStore else None + # codeAccVgprRead = deepcopy(writer.codes.codeAccVgprRead) if writer.states.serializedStore else None + codeAccVgprWrite = deepcopy(writer.codes.accVgprWrite) if writer.states.serializedStore else None + + module.add(self.computeWorkspaceSrd(writer, kernel, sgpr(sCtaIdx), tmpSgpr)) + + for batchIdx in range(0, numBatches): + elementStartIdx = batchIdx * numElementsPerBatch + elementStopIdx = min(elementStartIdx + numElementsPerBatch, len(elements[edgeI])) + elementsThisBatch = elements[edgeI][elementStartIdx:elementStopIdx] + #print("BATCH[%u/%u]: elements[edgeI][%u:%u] VGPRs=%u" % (batchIdx, numBatches, elementStartIdx, elementStopIdx,numVgprsPerElement )) + # elementVgprs can be large and should be perfectly tuned to the number of available + # VGPRS. We do not want to accidentally overflow and grow the pool here: + + module.add(self.fixupBatch(writer, kernel, ss, batchIdx, edge, gwvw, \ + elementsThisBatch, writer.vgprs.addrD, writer.vgprs.addrC, \ + tmpVgpr, cvtVgprStruct, \ + elementSgprs, tmpSgpr, codeAccVgprRead, codeAccVgprWrite)) + # delay PreLoopVmcntCase code after globalWrite + # if self.canOptimizePreLoopLWVmcnt: + # kStr += PreLoopVmcntCaseStr + + # Finish one write path, reset currPreLoopVmcntCase to Undefined + # self.currPreLoopVmcntCase = PreLoopVmcntCase.Undefined + + # kStr += inst("s_branch", skStoreLabel, "jump to store") + + return module + + def fixupBatch(self, writer, kernel, ss, batchIdx, edge, gwvw, \ + batchElements, addrD, addrC, \ + tmpVgpr, tmpCVTVgpr, batchElementSgprs, tmpSgpr, codeAccVgprRead, codeAccVgprWrite): + module = Module("StreamK Common fixupBatch") + + module.addComment0("optSingleColVgpr=%u optSharedColVgpr=%u optSGPRUsage=%s optSrdIncForRow=%u" % \ + (ss.optSingleColVgpr, ss.optSharedColVgpr, ss.optSGPRUsage, ss.optSrdIncForRow)) + + if kernel["StoreSyncOpt"]: + module.add(SSleep(kernel["StoreSyncOpt"] - 1, "optimization: sync and wait")) + module.add(SBarrier()) + + # comment tt1, tt0, vc1, vc0 + # tt = thread tile, vc=vector component + commentStr = "Fixup%s Batch #%u (d1,d0,vc1,vc0) =\n " \ + % (" Edge" if edge else "", batchIdx) + for elementIdx in range(0, len(batchElements)): + element = batchElements[elementIdx] + commentStr += "(%u,%u,%u,%u:vw%u)" % (element[0], element[1], element[2], element[3], gwvw) + if elementIdx < len(batchElements)-1: + commentStr += "; " + module.addComment2(commentStr) + # print(self.kernelName) + # print(commentStr) + + # allow expanding vgpr pool for OptNLL + # preventOverflow = True #(not isOptNLL) + # ss.setupStoreElementsForBatch(kernel, gwvw, batchElements, batchElementSgprs, preventOverflow=preventOverflow, isWorkspace=True) + ss.setupStoreElementsForBatch(kernel, gwvw, batchElements, batchElementSgprs, isOptNLL=False, biasDim=0, isWorkspace=True) + + loadsIssued = 0 + storesIssued = 0 + tmpS01 = tmpSgpr # scratch sgprs + + wavelen = kernel["WavefrontSize"] + # laneSGPRC = writer.states.laneSGPRCount + # always use gwvw for buffer load C for atomic_cmpswap + # bpm = self.bpeCexternal * atomicW + # bpm = self.bpeCexternal * gwvw + # vgprLoadDW = 1*(bpm//4) + # atomic oparation width. 1 for b32, 2 for b64 + # atomicOpW = (atomicW * self.bpeCexternal) // 4 + # if atomicOpW > 2: + # # should not exceeding 2. + # atomicOpW = 2 + + ######################################## + # calculate addr and masks + module.addComment1("calc coords, apply mask, and issue loads (if necessary)") + # On input, coord0 and coord1 are VGPRs computed in the pre-batch code, based + # on the thread and tid number. These are ELEMENT offsets from start of tensor C + # for the top-left corner this thread will write. These are not changed + # across all the store loop iters. + if writer.db["ConservativeWaitCnt"] & 0x10: + module.add(SBarrier("debug")) + module.add(SWaitCnt(vmcnt=0, comment="ConservativeWaitCnt")) + if writer.states.archCaps["SeparateVscnt"]: + module.add(SWaitCnt(vscnt=0, comment="writes")) + module.add(SBarrier("debug")) + if not edge and writer.db["ForceEdgeStores"]>=2: + module.add(self.parentWriter.getBomb()) # should not get here + if edge and writer.db["AssertNoEdge"]: + module.add(self.parentWriter.getBomb()) # should not get here + + # atomicAddC = kernel["AtomicAddC"] and not edge + + ## create code Module to push mov vgpr,acc instructions + # if kernel["StoreCInUnroll"] and not edge: + # accVgprRead = Code.Module("movaccVgpr") + # self.StoreCUnrollLoadCWaitComment = "waitcnt for LoadC" # this will be used later to identify waitcnt for loadC + + for elementIdx in range(0, len(batchElements)): + element = batchElements[elementIdx] + addrCVgpr = ss.elementAddr[elementIdx].addrCVgpr + # addrDVgpr = ss.elementAddr[elementIdx].addrDVgpr + addrCalc = ss.elementAddr[elementIdx] + data = ss.elementData[elementIdx] + # mask = ss.elementMask[elementIdx] + # sumIdx = ss.elementSumIdx[elementIdx] + # d1 = element[0] + # d0 = element[1] + # vc1 = element[2] + vc0 = element[3] + + storeWidth = kernel["StoreVectorWidth"] + # storeWidth = 2 + if batchIdx == 0 and elementIdx == 0: + module.add(staticMultiply(vgpr(addrCVgpr), vgpr("Serial"), storeWidth * writer.states.bpeCinternal, sgpr(tmpS01))) + # kStr += inst("v_mul_lo_u32", , "Partials buffer address") + module.add(SMovB32(dst=sgpr(tmpS01), src=0, comment="Init sgpr offset")) + else: + increment = (kernel["WavefrontSize"] * 4) * storeWidth * writer.states.bpeCinternal + module.add(SAddU32(dst=sgpr(tmpS01), src0=sgpr(tmpS01), src1=increment, comment="Inc sgpr offset")) + + module.add(writer.readInput(kernel, ss, 'WS', kernel["ProblemType"]["ComputeDataType"], addrCalc, vc0, data, gwvw, addrCVgpr, sgpr(tmpS01))) + loadsIssued += 1 + + ######################################## + # AccVgpr read + # if kernel.enabledSetPrioSplitLDS: + # kStr += inst("s_setprio", "0", "") + if codeAccVgprRead is not None: + regsPerScalar = writer.states.bpeCinternal // writer.states.bpr # register per scalar + # loop over store instructions within one batch + for elementIdx in range(0, len(batchElements)): + # loop over scalars within one store instruction + for vi in range(0, gwvw): + # loop over registers within one scalar + for rIdx in range(0, regsPerScalar): + module.add(replaceHolder(codeAccVgprRead.items().pop(0), ss.elementSumIdx[elementIdx]*regsPerScalar + regsPerScalar*vi + rIdx - writer.states.c.startVgprValu)) + # tempStr = str(codeAccVgprRead.items().pop(0)) + # kStr += tempStr.replace("__placeholder__", str(ss.elementSumIdx[elementIdx]*regsPerScalar + regsPerScalar*vi + rIdx)) + # if kernel["StoreCInUnroll"] and not edge: + # tempStr = tempStr.replace("__placeholder__",str(elementIdx*gwvw*regsPerScalar + regsPerScalar*vi + rIdx)) + # accVgprRead.addCode(tempStr.replace("ValuC","L2GC")) + + if not kernel["MIArchVgpr"]: + module.add(SNop(1, "2 wait states required before reading vgpr")) + + ######################################## + # Not Atomic + ######################################## + # edge has v_cndmask so loads or stores may not issue, hard to track vmcnt: + interleaveStoreVmcnt = writer.states.interleaveStoreVmcnt and not edge + for elementIdx in range(0, len(batchElements)): + for vi in range(0, gwvw): + sumIdxV = ss.elementSumIdx[elementIdx] + vi + # covers sgemm, gemm_ex(HHS/HSS/BBS/BSS (HPA=T)), int8 (int8x4?) + if kernel["ProblemType"]["ComputeDataType"].isInt32() or kernel["ProblemType"]["ComputeDataType"].isSingle(): + if writer.db["ForceExpectedValue"]: + module.add(VMovB32(dst=vgpr("ValuC+%u"%sumIdxV), src=writer.db["ValueCExpectedValue"], comment="force expected value")) + if writer.db["ForceVSerial"]: + module.add(VMovB32(dst=vgpr("ValuC+%u"%sumIdxV), src=vgpr("Serial"), comment="force expected value to serial")) + if writer.db["CheckValueC"]: + module.add(SMovB32(dst=sgpr(tmpS01), src=writer.db["ValueCExpectedValue"], comment="Move expected value")) + module.add(writer.getCmpAssert(writer.asmAssert.eq, vgpr("ValuC+%u"%sumIdxV), sgpr(tmpS01))) + + ######################################## + # wait for batched load + if not interleaveStoreVmcnt: # beta and + module.add(SWaitCnt(vmcnt=0, comment="wait C")) + if writer.states.archCaps["SeparateVscnt"]: + module.add(SWaitCnt(vscnt=0, comment="writes")) + + # PreLoop LWVmcnt: When a vmcnt(cnt) is inserted here, means the GlobalLoad for PAP is finished + # So the preLoopVmcntDict value is meaningless since we no longer need to wait in next PreLoop + # And this only occurs when beta=true, so case must not be 2 or 3 + # assert self.currPreLoopVmcntCase not in self.preLoopVmcntDict, \ + # "PreLoopVmcntCase 2 or 3 shouldn't enter the beta true case" + + module.addComment1("apply mask, calc new C and issue writes") + #kStr += self.bomb() # can see store addresses just before the store inst + + # if kernel["ProblemType"]["DestDataType"].isBFloat16() and kernel["ProblemType"]["HighPrecisionAccumulate"]: + # vgprBf16Temp = tmpCVTVgpr + # vgprBf16Mask = vgprBf16Temp + 1 + # vgprFp32Nan = vgprBf16Temp + 2 + # vgprBf16Inc = vgprBf16Temp + 3 + # kStr += inst("v_mov_b32", vgpr(vgprBf16Mask), "0xffff0000", "mask for pack two bfloat16 element to 32bit" ) + # kStr += inst("v_mov_b32", vgpr(vgprFp32Nan), "0x7fff0000", "fp32 Nan" ) + # kStr += inst("v_mov_b32", vgpr(vgprBf16Inc), "0x7fff", "rounding bias for bfloat16" ) + if kernel["ProblemType"]["DestDataType"].isBFloat16() and kernel["ProblemType"]["HighPrecisionAccumulate"]: + module.add(VMovB32(vgpr(cvtVgprStruct.vgprBf16Mask), "0xffff0000", "mask for pack two bfloat16 element to 32bit" )) + module.add(VMovB32(vgpr(cvtVgprStruct.vgprFp32Nan), "0x7fff0000", "fp32 Nan" )) + module.add(VMovB32(vgpr(cvtVgprStruct.vgprBf16Inc), "0x7fff", "rounding bias for bfloat16" )) + elif kernel["ProblemType"]["DestDataType"].isFloat8() and kernel["ProblemType"]["HighPrecisionAccumulate"]: + module.add(VMovB32(vgpr(cvtVgprStruct.vgprFp8NanInf), "0x207", "Nan and +/- inf" )) + module.add(VMovB32(vgpr(cvtVgprStruct.vgprFp8Max), "0x43700000", "Fp8 Max value 240 as float32" )) + module.add(VMovB32(vgpr(cvtVgprStruct.vgprFp8Min), "0xc3700000", "Fp8 Min value -240 as float32" )) + elif kernel["ProblemType"]["DestDataType"].isBFloat8() and kernel["ProblemType"]["HighPrecisionAccumulate"]: + module.add(VMovB32(vgpr(cvtVgprStruct.vgprBF8NanInf), "0x207", "Nan and +/- inf" )) + module.add(VMovB32(vgpr(cvtVgprStruct.vgprBF8Max), "0x47600000", "BF8 Max value 57344 as float32" )) + module.add(VMovB32(vgpr(cvtVgprStruct.vgprBF8Min), "0xc7600000", "BF8 Min value -57344 as float32" )) + + # DestDataType for 8bit Float can only be F8 or B8 + # if kernel["ProblemType"]["DestDataType"].isFloat8() or kernel["ProblemType"]["DestDataType"].isBFloat8(): # F8 is always HPA + # # make vgprF8Temp0 always even to use pk instruction later + # if tmpCVTVgpr % 2 == 0: + # vgprF8Temp0 = tmpCVTVgpr + # vgprF8Temp1 = vgprF8Temp0 + 1 + # vgprF8Max = vgprF8Temp0 + 2 + # vgprF8Min = vgprF8Temp0 + 3 + # else: + # vgprF8Max = tmpCVTVgpr + # vgprF8Temp0 = vgprF8Max + 1 + # vgprF8Temp1 = vgprF8Max + 2 + # vgprF8Min = vgprF8Max + 3 + + # if kernel["ProblemType"]["Fp32toFp8SWClip"]: + # # set flag of f32 NaN and +/- INF for v_cmp_class + # vgprFp32NanInfFlag = vgprF8Min + 1 + # kStr += inst("v_mov_b32", vgpr(vgprFp32NanInfFlag), "0x207", "flag for Nan and +/- inf" ) + # # set max/min values for clipping + # if kernel["ProblemType"]["DestDataType"].isFloat8(): + # kStr += inst("v_mov_b32", vgpr(vgprF8Max), "0x43700000", "save 240.0f as max for clipping" ) + # kStr += inst("v_mov_b32", vgpr(vgprF8Min), "0xC3700000", "save -240.0f as min for clipping" ) + # else: #BFloat8 + # kStr += inst("v_mov_b32", vgpr(vgprF8Max), "0x47600000", "save 57344.0f as max for clipping" ) + # kStr += inst("v_mov_b32", vgpr(vgprF8Min), "0xC7600000", "save -57344`.0f as min for clipping" ) + + for elementIdx in range(0, len(batchElements)): + element = batchElements[elementIdx] + addr = ss.elementAddr[elementIdx].addrDVgpr + mask = ss.elementMask[elementIdx] + addrCalc = ss.elementAddr[elementIdx] + # d1 = element[0] + # d0 = element[1] + # vc1 = element[2] + vc0 = element[3] + sumIdx = ss.elementSumIdx[elementIdx] + + # apply in-bounds exec mask + if edge and not kernel["BufferStore"]: + module.add(writer.getEdgeMovInstType()(EXEC(), sgpr(mask, writer.states.laneSGPRC), "sgprs -> exec")) + # kStr += inst("s_mov_b{}".format(wavelen), self.exec, sgpr(mask,laneSGPRC), "sgprs -> exec" ) + + # if beta: + # if GWVW=1 the half path still assumes we have + # at least two stores so does some combining across VI - + # for example assuming we can have two elements and can use pk_mul + # here: + if interleaveStoreVmcnt: # beta and + if writer.states.archCaps["SeparateVscnt"]: + vmcnt = loadsIssued - elementIdx - 1 + vmComment = "{} = {} - {} - 1".format(vmcnt, loadsIssued, elementIdx) + else: + waitStoreCnt = storesIssued if not kernel["GroupLoadStore"] else 0 + vmcnt = loadsIssued - elementIdx + waitStoreCnt - 1 + vmComment = "{} = {} - {} + {} - 1".format(vmcnt, loadsIssued, elementIdx, waitStoreCnt) + + maxVmcnt = writer.states.asmCaps["MaxVmcnt"] + vmcnt = min(vmcnt, maxVmcnt) + #print "wmvcnt=", vmcnt + module.addSpaceLine() + # if not atomicAddC: + module.add(SWaitCnt(vmcnt=vmcnt, comment="wait C (interleaved) {}".format(vmComment))) + + # PreLoop LWVmcnt: When a vmcnt(cnt) is inserted here, means the GlobalLoad for PAP is finished + # So the preLoopVmcntDict value is meaningless since we no longer need to wait in next PreLoop + # And this only occurs when beta=true, so case must not be 2 or 3 + # assert self.currPreLoopVmcntCase not in self.preLoopVmcntDict, "PreLoopVmcntCase 2 or 3 shouldn't enter the beta true case" + + for vi in range(0, gwvw): + dataV = ss.elementData[elementIdx] + int(vi*ss.cfg.numVgprsPerDataPerVI) + sumIdxV = ss.elementSumIdx[elementIdx] + vi + if kernel["ProblemType"]["ComputeDataType"].isHalf(): + if not kernel["ProblemType"]["HighPrecisionAccumulate"]: + if writer.states.asmCaps["HasWMMA"] and kernel["EnableMatrixInstructionStore"]: + dataV = ss.elementData[elementIdx] + int(vi / 2 * ss.cfg.numVgprsPerDataPerVI) + # if (vi % 2) == 0: + # kStr += inst("v_pk_mul_f16", vgpr(dataV), sgpr("Beta"), vgpr(dataV+0), \ + # "%s = C*beta ei=%u vi=%u"%(vgpr(dataV),elementIdx, vi)) + # else: + if (vi % 2) != 0: + module.add(VLShiftRightB32(dst=vgpr(dataV), shiftHex=16, src=vgpr(dataV), \ + comment="shift 16bit to get next half of packed ValueC")) + # dataV+0 = new c = old c*beta + rC + module.add(VAddPKF16(dst=vgpr("ValuC+%u"%(sumIdxV)), src0=vgpr(dataV), src1=vgpr("ValuC+%u"%(sumIdxV)), \ + comment="sum*alpha + C*beta")) + elif sumIdxV%2==0 or (not ss.cfg.halfDataRegPerVI and gwvw==1): + newSumIdxV = sumIdxV // 2 - writer.states.c.startVgprValu + # dataV+0 = new c = old c*beta + # kStr += inst("v_pk_mul_f16", vgpr(dataV), sgpr("Beta"), vgpr(dataV+0), \ + # "%s = C*beta ei=%u vi=%u"%(vgpr(dataV),elementIdx, vi)) + # dataV+0 = new c = old c*beta + rC + module.add(VAddPKF16(dst=vgpr("ValuC+%u"%(newSumIdxV)), src0=vgpr(dataV), src1=vgpr("ValuC+%u"%(newSumIdxV)), \ + comment="sum*alpha + C*beta")) + else: + pass # add will have been done previously + else: # HPA + newSumIdxV = sumIdxV - writer.states.c.startVgprValu + # dataV+0 = new c = old c*beta + rC + # src0 = beta = f32 = opsel 00 + # src1 = dataV = f16.lo = opsel 10 or 11 depending on even/odd + # src2 = sumIdxV = f32 = opsel 00 + dataCExternal = ss.elementData[elementIdx] + vi//2 + hi16 = (vi + gwvw*vc0) % 2 + # TODO try to replace with add? need opsel for f16 src + # kStr += inst(self.mixinst, vgpr("ValuC+%u"%sumIdxV), sgpr("Beta"), \ + # module.add(writer.states.mixinst(dst=vgpr("ValuC+%u"%newSumIdxV), src0=sgpr("Beta"), \ + # src1=vgpr(dataCExternal), src2=vgpr("ValuC+%u"%newSumIdxV), \ + # vop3=VOP3PModifiers(op_sel=[0,hi16,0], op_sel_hi=[0,1,0]), + # comment="//C*=beta")) + module.add(writer.states.mixinst(dst=vgpr("ValuC+%u"%newSumIdxV), src0=1, \ + src1=vgpr(dataCExternal), src2=vgpr("ValuC+%u"%newSumIdxV), \ + vop3=VOP3PModifiers(op_sel=[0,hi16,0], op_sel_hi=[0,1,0]), + comment="//C*=beta")) + # kStr += inst(self.mixinst, vgpr("ValuC+%u"%sumIdxV), 1, \ + # vgpr(dataCExternal), vgpr("ValuC+%u"%sumIdxV), \ + # "op_sel:[0,%u,0] op_sel_hi:[0,1,0]" % (hi16), \ + # "//C*=beta") + + elif kernel["ProblemType"]["ComputeDataType"].isBFloat16(): + if kernel["ProblemType"]["HighPrecisionAccumulate"]: + # dataV+0 = new c = old c*beta + rC + # src0 = beta = f32 = opsel 00 + # src1 = dataV = f16.lo = opsel 10 or 11 depending on even/odd + # src2 = sumIdxV = f32 = opsel 00 + dataCExternal = ss.elementData[elementIdx] + vi//2 + # if (vi%2) == 1: + # kStr += inst("v_and_b32", vgpr(tmpVgpr), vgpr(dataCExternal), vgpr(vgprBf16Mask), "convert bf16 to fp32") + # else: + # kStr += inst("v_lshlrev_b32", vgpr(tmpVgpr), "16", vgpr(dataCExternal), "convert bf16 to fp32" ) + module.add(VCvtBF16toFP32(dst=(tmpVgpr), src=(dataCExternal), vgprMask=(cvtVgprStruct.vgprBf16Mask), vi=(vi))) + newSumIdxV = sumIdxV - writer.states.c.startVgprValu + module.add(VAddF32(dst=vgpr("ValuC+%u"%sumIdxV), src0=vgpr("ValuC+%u"%sumIdxV), src1=vgpr(tmpVgpr), comment="accum partials")) + + # float8 precision + elif kernel["ProblemType"]["DestDataType"].isFloat8(): + if kernel["ProblemType"]["HighPrecisionAccumulate"]: + newSumIdxV = sumIdxV - self.parentWriter.states.c.startVgprValu + # Generate single f32 code if edge is detected. + isPK = False + if ((vi + 1) == self.gwvw) and ((self.gwvw % 2) == 1): + sb = SelectBit.BYTE_0 if self.gwvw == 1 else SelectBit.BYTE_2 + module.add(VCvtFP8toF32(dst=vgpr(tmpVgpr), src=vgpr(dataV), sdwa=SDWAModifiers(src0_sel=sb))) + # Original packed route + elif vi%2 == 1: + continue + else: + isPK = True + sb = SelectBit.WORD_0 if vi == 0 else SelectBit.WORD_1 + module.add(VCvtPkFP8toF32(dst=vgpr(tmpVgpr, 2), src=vgpr(dataV), sdwa=SDWAModifiers(src0_sel=sb))) + module.add(SNop(waitState=0)) + if kernel["ProblemType"]["ComputeDataType"].isSingle(): + module.add(VAddF32(dst=vgpr("ValuC+%u"%newSumIdxV), src0=vgpr("ValuC+%u"%newSumIdxV), src1=vgpr(tmpVgpr), comment="accum partials")) + if isPK: + module.add(VAddF32(dst=vgpr("ValuC+%u"%(newSumIdxV+1)), src0=vgpr("ValuC+%u"%(newSumIdxV+1)), src1=vgpr(tmpVgpr+1), comment="accum partials")) + + # bfloat8 precision + elif kernel["ProblemType"]["DestDataType"].isBFloat8(): + if kernel["ProblemType"]["HighPrecisionAccumulate"]: + newSumIdxV = sumIdxV - self.parentWriter.states.c.startVgprValu + # Generate single f32 code if edge is detected. + isPK = False + if ((vi + 1) == self.gwvw) and ((self.gwvw % 2) == 1): + sb = SelectBit.BYTE_0 if self.gwvw == 1 else SelectBit.BYTE_2 + module.add(VCvtBF8toF32(dst=vgpr(tmpVgpr), src=vgpr(dataV), sdwa=SDWAModifiers(src0_sel=sb))) + # Original packed route + elif vi%2 == 1: + continue + else: + isPK = True + sb = SelectBit.WORD_0 if vi == 0 else SelectBit.WORD_1 + module.add(VCvtPkBF8toF32(dst=vgpr(tmpVgpr, 2), src=vgpr(dataV), sdwa=SDWAModifiers(src0_sel=sb))) + module.add(SNop(waitState=0)) + if kernel["ProblemType"]["ComputeDataType"].isSingle(): + module.add(VAddF32(dst=vgpr("ValuC+%u"%newSumIdxV), src0=vgpr("ValuC+%u"%newSumIdxV), src1=vgpr(tmpVgpr), comment="accum partials")) + if isPK: + module.add(VAddF32(dst=vgpr("ValuC+%u"%(newSumIdxV+1)), src0=vgpr("ValuC+%u"%(newSumIdxV+1)), src1=vgpr(tmpVgpr+1), comment="accum partials")) + + elif kernel["ProblemType"]["ComputeDataType"].isSingle(): + newSumIdxV = sumIdxV - writer.states.c.startVgprValu + module.add(VAddF32(dst=vgpr("ValuC+%u"%newSumIdxV), src0=vgpr("ValuC+%u"%newSumIdxV), src1=vgpr(dataV+0), comment="accum partials")) + + elif kernel["ProblemType"]["ComputeDataType"].isInt32(): + newSumIdxV = sumIdxV - writer.states.c.startVgprValu + # assume we will need to replace v_mac_f32 with v_add_u32 and s_mul_lo_i32 + # v_mad_i32_i24 + module.add(VAddU32(dst=vgpr("ValuC+%u"%newSumIdxV), src0=vgpr(dataV+0), src1=vgpr("ValuC+%u"%newSumIdxV), comment="accum partials")) + + elif kernel["ProblemType"]["ComputeDataType"].isDouble(): + newSumIdxV = sumIdxV * 2 - writer.states.c.startVgprValu + # dataV+0 = new c = old c*beta + module.add(VAddF64(dst=vgpr("ValuC+%u"%(newSumIdxV*2),2), src0=vgpr("ValuC+%u"%(newSumIdxV*2),2), src1=vgpr(dataV+0,2), comment="accum partials")) + + # single precision complex + elif kernel["ProblemType"]["ComputeDataType"].isSingleComplex(): + newSumIdxV = sumIdxV * 2 - writer.states.c.startVgprValu + module.add(VAddF32(dst=vgpr("ValuC+%u"%(newSumIdxV*2)), src0=vgpr("ValuC+%u"%(newSumIdxV*2)), src1=vgpr(dataV+0), comment="accum partials real")) + module.add(VAddF32(dst=vgpr("ValuC+%u"%(newSumIdxV*2+1)), src0=vgpr("ValuC+%u"%(newSumIdxV*2+1)), src1=vgpr(dataV+1), comment="accum partials imag")) + + # double precision complex + elif kernel["ProblemType"]["ComputeDataType"].isDoubleComplex(): + newSumIdxV = sumIdxV * 4 - self.parentWriter.states.c.startVgprValu + module.add(VAddF64(dst=vgpr("ValuC+%u"%(newSumIdxV*4+0),2), src0=vgpr("ValuC+%u"%(newSumIdxV*4+0),2), src1=vgpr(dataV+0,2), comment="accum partials real")) + module.add(VAddF64(dst=vgpr("ValuC+%u"%(newSumIdxV*4+2),2), src0=vgpr("ValuC+%u"%(newSumIdxV*4+2),2), src1=vgpr(dataV+2,2), comment="accum partials imag")) + + ######################################## + # AccVgpr write + # if kernel.enabledSetPrioSplitLDS: + # kStr += inst("s_setprio", "0", "") + if codeAccVgprWrite is not None: + regsPerScalar = writer.states.bpeCinternal // writer.states.bpr # register per scalar + # loop over store instructions within one batch + for elementIdx in range(0, len(batchElements)): + # loop over scalars within one store instruction + for vi in range(0, gwvw): + # loop over registers within one scalar + for rIdx in range(0, regsPerScalar): + module.add(replaceHolder(codeAccVgprWrite.items().pop(0), ss.elementSumIdx[elementIdx]*regsPerScalar + regsPerScalar*vi + rIdx - writer.states.c.startVgprValu)) + # tempStr = str(codeAccVgprWrite.items().pop(0)) + # kStr += tempStr.replace("__placeholder__", str(ss.elementSumIdx[elementIdx]*regsPerScalar + regsPerScalar*vi + rIdx)) + # if kernel["StoreCInUnroll"] and not edge: + # tempStr = tempStr.replace("__placeholder__",str(elementIdx*gwvw*regsPerScalar + regsPerScalar*vi + rIdx)) + # accVgprRead.addCode(tempStr.replace("ValuC","L2GC")) + + if not kernel["MIArchVgpr"]: + module.add(SNop(1, "2 wait states required before reading vgpr")) + + #kStr += self.bomb(5) + # if self.db["CheckStoreC"]>=0: + # useBuffer = kernel["BufferStore"] + # # Note - CheckStoreC won't work for EDGE store cases since they load 0 for OOB, would need more sophisticated check + # # Note - TODO- CheckStoreC also won't work for StoreRemap + # kStr += inst("s_waitcnt", "vmcnt(0)", "CheckStoreC, wait for stores to complete" ) + # if self.archCaps["SeparateVscnt"]: + # kStr += inst("s_waitcnt_vscnt", "null", "0", "writes") + # for elementIdx in range(0, len(batchElements)): + # addr = ss.elementAddr[elementIdx].addrDVgpr + # sumIdx = ss.elementSumIdx[elementIdx] + + # bps = kernel["ProblemType"]["DestDataType"].numBytes() * gwvw + # if kernel["BufferStore"]: + # addr0 = vgpr(addr) + # addr1 = sgpr("SrdC", 4) + # else: + # addr0 = vgpr(addr,2) + # addr1 = "" + + # if kernel["ProblemType"]["DestDataType"].isHalf() or kernel["ProblemType"]["DestDataType"].isBFloat16(): + # if not kernel["ProblemType"]["HighPrecisionAccumulate"]: + # kStr += self.chooseGlobalRead(useBuffer, bps, sumIdx//2, \ + # addr0, addr1, soffset=0, offset=0, extraFields="", dtlNoDestVgpr=False, hi16=sumIdx%2).toStr() + # else: + # kStr += self.chooseGlobalRead(useBuffer, bps, sumIdx, \ + # addr0, addr1, soffset=0, offset=0, extraFields="", dtlNoDestVgpr=False, hi16=0).toStr() + # elif kernel["ProblemType"]["DestDataType"].isInt32() or kernel["ProblemType"]["DestDataType"].isSingle(): + # kStr += self.chooseGlobalRead(useBuffer, bps, sumIdx, \ + # addr0, addr1, soffset=0, offset=0, extraFields="", dtlNoDestVgpr=False).toStr() + # elif kernel["ProblemType"]["DestDataType"].isDouble() or kernel["ProblemType"]["DestDataType"].isSingleComplex() : + # kStr += self.chooseGlobalRead(useBuffer, bps, sumIdx*2, \ + # addr0, addr1, soffset=0, offset=0, extraFields="", dtlNoDestVgpr=False).toStr() + # elif kernel["ProblemType"]["DestDataType"].isDoubleComplex(): + # kStr += self.chooseGlobalRead(useBuffer, bps, sumIdx*4, \ + # addr0, addr1, soffset=0, offset=0, extraFields="", dtlNoDestVgpr=False).toStr() + # kStr += inst("s_waitcnt", "vmcnt(0)", "CheckStoreC, wait for stores to complete" ) + # if self.archCaps["SeparateVscnt"]: + # kStr += inst("s_waitcnt_vscnt", "null", "0", "writes") + + # # Add checks for expected values: + # kStr += inst("s_mov_b32", sgpr(tmpS01), self.db["CheckStoreC"], "expected value") + # for elementIdx in range(0, len(batchElements)): + # sumIdx = ss.elementSumIdx[elementIdx] + # # Need to fix for other types: + # assert (kernel["ProblemType"]["DestDataType"].isSingle() or kernel["ProblemType"]["DestDataType"].isInt32()) + # kStr += self.assert_eq(vgpr(sumIdx), sgpr(tmpS01)) + + + if edge and (not kernel["BufferStore"]): # atomic or + # subsequent batch must start with full exec mask + # BufferStore doesn't need exec since it used buffer range checking when + # possible + module.add(self.getEdgeMovInstType()(EXEC(), -1, "full mask -> exec")) + + if writer.db["ConservativeWaitCnt"] & 0x40: + module.add(SBarrier("debug")) + module.add(SWaitCnt(vmcnt=0, vscnt=0, comment="ConservativeWaitCnt")) + module.add(SBarrier("debug")) + ######################################## + # End Not Atomic + ######################################## + + # return registers to pool: + lastData = -1 + for elementIdx in range(0, len(batchElements)): + if not ss.sharedColDVgprs: + addrCalc: AddrCalculation = ss.elementAddres[elementIdx] + addrDVgpr = addrCalc.addrDVgpr + addrCVgpr = addrCalc.addrCVgpr + writer.vgprPool.checkIn(addrDVgpr) + if addrCVgpr != addrDVgpr: + writer.vgprPool.checkIn(addrCVgpr) + + data = ss.elementData[elementIdx] + if data != 0: + if data != lastData: + writer.vgprPool.checkIn(data) + lastData = data + + ss.firstBatch = False + ss.checkInTempVgprC() + + if writer.states.serializedStore: + module.add(SNop(0, "1 wait state required when next inst writes vgprs held by previous dwordx4 store inst")) + + # Update the store cnt to preLoopVmcntDict for Case2/3 + # (No need to update for Case0:'Undefined' or Case4:'OrdNLL_B1_Store') + # if self.currPreLoopVmcntCase in self.preLoopVmcntDict: + # if not self.archCaps["SeparateVscnt"]: + # self.preLoopVmcntDict[self.currPreLoopVmcntCase] += storesIssued + + return module class StreamKOff(StreamK): kernel = {"StreamK": 0} @@ -872,7 +1596,7 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr return module - def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements): + def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct): module = Module("StreamK Off storeBranches") return module @@ -942,9 +1666,9 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) return module - def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements): + def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct): module = Module("StreamK Basic storeBranches") - module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) + module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct)) return module def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel): @@ -1052,9 +1776,9 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) return module - def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements): + def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct): module = Module("StreamK TwoTileOriginal storeBranches") - module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) + module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, element, tmpVgpr, cvtVgprStruct)) return module def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel): @@ -1198,9 +1922,9 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr module.add(self.calculateLoopNumIterCommon(writer, kernel, loopCounterName, loopIdx, tmpSgprInfo)) return module - def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements): + def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct): module = Module("StreamK TwoTileDPFirst storeBranches") - module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, elements)) + module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct)) return module def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel): diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 2ac36d1802..99a8c1ed10 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -327,6 +327,7 @@ def __post_init__(self): @dataclass class CodeModules: accVgprRead: Optional[Module] = None + accVgprWrite: Optional[Module] = None mulAlphaMultipleBuffer: Optional[Module] = None mulAlphaOther: Optional[Module] = None localWriteA: Optional[Module] = None diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 44b2e76a9d..d91510269e 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -4436,7 +4436,9 @@ def fixPreloadOffset(offset, sgpxIdxVec, numStoreSgprToLoad): #instCycles = kernel["MatrixInstM"] // 2 # 32x32 is 64 cycles, 16x16 is 32 cycles, 4x4 is 8 cycles #module.add(SNop(waitState=instCycles)) module.addComment1("Mapping of Acc register -> C Vgpr register") - self.codes.accVgprRead = mapAcctoArchRegs(kernel) + self.codes.accVgprRead = mapAcctoArchRegs(kernel, write=False) + if kernel["StreamK"] > 0 and kernel["StreamKAtomic"] == 0: + self.codes.accVgprWrite = mapAcctoArchRegs(kernel, write=True) if kernel["MIArchVgpr"]: module.addComment1("Multiply MI out register with Alpha -> C Vgpr register") self.codes.mulAlphaMultipleBuffer = moveMIoutToArch(kernel, self.states.startVgprAlphaTmp) @@ -8310,10 +8312,6 @@ def globalWriteElements(self, kernel, tPA, tPB, vectorWidths_2, vectorWidths_1, if not self.do["PostLoop"]: return Module("GlobalWriteElements (Empty)") module = Module("GlobalWriteElements") - skPartialsLabel = Label(label=self.labels.getNameInc("SK_Partials"), comment="") - skComponent = Component.StreamK.find(self) - module.add(skComponent.storeBranches(self, kernel, skPartialsLabel, vectorWidths_1, elements_1)) - module.addComment2("Global Write Elements") if self.states.numStoreSgprToLoad or self.states.numStoreSgprToLoad2: # Wait for kernel args module.add(SWaitCnt(lgkmcnt=0, comment="wait for %u bytes of kern args."%((self.states.numStoreSgprToLoad+self.states.numStoreSgprToLoad2) * 4))) @@ -8768,6 +8766,9 @@ def globalWriteElements(self, kernel, tPA, tPB, vectorWidths_2, vectorWidths_1, # branch B1 or B0 # betaLabel = Label("GW_Beta", "") if (gsuLimit > 1) and (kernel["GlobalSplitU"] > 1) else Label(self.labels.getNameInc("GW_Beta"), "") betaLabel = Label(self.labels.getNameInc("GW_Beta"), "") + skPartialsLabel = Label(label=self.labels.getNameInc("SK_Partials"), comment="") + skComponent = Component.StreamK.find(self) + module.add(skComponent.storeBranches(self, kernel, skPartialsLabel, vectorWidths_1, elements_1, tmpVgpr, cvtVgprStruct)) betaModules = Module("Betas") currentInstLength = 0 @@ -9594,9 +9595,14 @@ def readInput(self, kernel, ss, tc: str, dataType, addrCalc, vc0, data, gwvw, ad isSlc = kernel["NonTemporal%s"%tc] & 0x2 isNT = kernel["NonTemporal%s"%tc] & 0x4 + soffset = 0 if tc == 'E': - globalOffset = addrCalc.globalOffsetE - bpeType = self.states.bpeE + globalOffset = addrCalc.globalOffsetE + bpeType = self.states.bpeE + elif tc == 'WS': + soffset = tmpS01 + globalOffset = 0 + bpeType = self.states.bpeCinternal else: if dataType == kernel["ProblemType"]["ComputeDataType"]: globalOffset = addrCalc.globalOffsetInternal @@ -9608,22 +9614,22 @@ def readInput(self, kernel, ss, tc: str, dataType, addrCalc, vc0, data, gwvw, ad if kernel["_GlobalAccumulation"] == "MultipleBufferSingleKernel": globalOffset = int((globalOffset/self.states.bpeCexternal) * self.states.bpr * kernel["ProblemType"]["DestDataType"].numRegisters()) - if ss.optSrdIncForRow and addrCalc.rowInc: + isWorkspace = tc == 'WS' + if ss.optSrdIncForRow and addrCalc.rowInc and not isWorkspace: module.add(addrCalc.incrementToNextRow(kernel, tc, ss, tmpS01, bpeType=bpeType)) if dataType.isHalf(): hi16 = 0 if self.states.HHH_WMMA else (vc0 % 2) module.add(self.chooseGlobalRead(useBuffer, bps, data, \ - addr0, addr1, soffset=0, offset=globalOffset, \ - glc=isGlc, slc=isSlc, nt=isNT, lds=False, - hi16=hi16, - comment="load %s"%tc)) + addr0, addr1, soffset=soffset, offset=globalOffset, \ + glc=isGlc, slc=isSlc, nt=isNT, lds=False, hi16=hi16, \ + comment="load %s"%tc)) elif dataType.isInt8() or dataType.is8bitFloat(): module.add(self.chooseGlobalRead(useBuffer, bps, data, \ - addr0, addr1, soffset=0, offset=globalOffset, \ - glc=isGlc, slc=isSlc, nt=isNT, lds=False, \ - #hi16=vc0 % 4, - comment="load %s"%tc)) + addr0, addr1, soffset=soffset, offset=globalOffset, \ + glc=isGlc, slc=isSlc, nt=isNT, lds=False, \ + #hi16=vc0 % 4, + comment="load %s"%tc)) elif dataType.isBFloat16() or \ dataType.isInt32() or \ dataType.isSingle() or \ @@ -9631,7 +9637,7 @@ def readInput(self, kernel, ss, tc: str, dataType, addrCalc, vc0, data, gwvw, ad dataType.isSingleComplex() or \ dataType.isDoubleComplex(): module.add(self.chooseGlobalRead(useBuffer, bps, data, \ - addr0, addr1, soffset=0, offset=globalOffset, \ + addr0, addr1, soffset=soffset, offset=globalOffset, \ glc=isGlc, slc=isSlc, nt=isNT, lds=False, \ comment="load %s"%tc)) diff --git a/tensilelite/Tensile/KernelWriterModules.py b/tensilelite/Tensile/KernelWriterModules.py index 7c87521317..dcd4a53a3a 100644 --- a/tensilelite/Tensile/KernelWriterModules.py +++ b/tensilelite/Tensile/KernelWriterModules.py @@ -187,11 +187,11 @@ def accVgprImagNumOffset(kernel): # MapAcctoArch # function to map MFMA Acc Registers to Arch VGPR register ############################################################################## -def mapAcctoArchRegs(kernel): +def mapAcctoArchRegs(kernel, write=False): acc2arch, _ = accToArchMapper(kernel) complexMultiplier = 2 if kernel["ProblemType"]["DataType"].isComplex() else 1 - imod = Module("AccVgprRead") + imod = Module("AccVgpr{}".format("Write" if write else "Read")) imod.itemList = [None] * kernel["MIRegPerOut"] * complexMultiplier * len(acc2arch) accImOffset = accVgprImagNumOffset(kernel) for i in range(len(acc2arch)): @@ -201,13 +201,23 @@ def mapAcctoArchRegs(kernel): srcIdx = ((i * kernel["MIRegPerOut"] + r) + (cm*accImOffset)) if not kernel["MIArchVgpr"]: accStr = accvgpr(srcIdx) - imod.itemList[destIdx] = VAccvgprReadB32(dst=vgpr(Holder(name="ValuC")), - src=accStr, - comment="copy acc to vreg[%u]" % destIdx) + if write: + imod.itemList[destIdx] = VAccvgprWriteB32(dst=accStr, + src=vgpr(Holder(name="ValuC")), + comment="copy vreg[%u] to acc" % destIdx) + else: + imod.itemList[destIdx] = VAccvgprReadB32(dst=vgpr(Holder(name="ValuC")), + src=accStr, + comment="copy acc to vreg[%u]" % destIdx) else: - imod.itemList[destIdx] = VMovB32(dst=vgpr(Holder(name="ValuC")), - src=vgpr("ValuC+%u"%srcIdx), - comment="copy MI out reg to vreg[%u]" % destIdx) + if write: + imod.itemList[destIdx] = VMovB32(dst=vgpr("ValuC+%u"%srcIdx), + src=vgpr(Holder(name="ValuC")), + comment="copy vreg[%u] to MI out reg" % destIdx) + else: + imod.itemList[destIdx] = VMovB32(dst=vgpr(Holder(name="ValuC")), + src=vgpr("ValuC+%u"%srcIdx), + comment="copy MI out reg to vreg[%u]" % destIdx) return imod ############################################################################## From 6a50f3db3bc75d1201f2c05b1bce3bb696ec8386 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Mon, 22 Jul 2024 21:59:20 +0000 Subject: [PATCH 17/36] Enable sk grid and fix tile calculation --- .../Tensile/Source/lib/source/ContractionProblem.cpp | 3 ++- .../Tensile/Source/lib/source/ContractionSolution.cpp | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp b/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp index 03e65486a1..72bc53b498 100644 --- a/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp +++ b/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp @@ -664,7 +664,8 @@ namespace Tensile numWG.x = CeilDivide(numWG.x, sizeMapping.macroTile.x); numWG.y = CeilDivide(numWG.y, sizeMapping.macroTile.y); - numWG.y *= sizeMapping.globalSplitU; + if(sizeMapping.streamK == 0) + numWG.y *= sizeMapping.globalSplitU; size_t problemTiles = numWG.x * numWG.y; if(sizeMapping.persistentKernelAlongBatch || sizeMapping.streamK != 0) diff --git a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp index 026308318e..6ab1439807 100644 --- a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp +++ b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp @@ -887,10 +887,9 @@ namespace Tensile if(sizeMapping.streamK != 0) { skGrid = getSKGrid(hardware, tiles); - // TODO enable grid size change once code gen changes are implemented - // rv.numWorkGroups.x = skGrid; - // rv.numWorkGroups.y = 1; - // rv.numWorkGroups.z = 1; + rv.numWorkGroups.x = skGrid; + rv.numWorkGroups.y = 1; + rv.numWorkGroups.z = 1; } } From 16fa50553f6158ba9125f268446c1a43c0c1db54 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Tue, 23 Jul 2024 20:16:16 +0000 Subject: [PATCH 18/36] Some build fixes --- .../Source/client/source/SolutionIterator.cpp | 4 ++-- .../Source/lib/source/ContractionProblem.cpp | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tensilelite/Tensile/Source/client/source/SolutionIterator.cpp b/tensilelite/Tensile/Source/client/source/SolutionIterator.cpp index 41cef1d2b6..67597c1424 100644 --- a/tensilelite/Tensile/Source/client/source/SolutionIterator.cpp +++ b/tensilelite/Tensile/Source/client/source/SolutionIterator.cpp @@ -113,7 +113,7 @@ namespace Tensile if(solution.requiredHostWorkspaceSizePerProblem == static_cast(-1)) { solution.requiredHostWorkspaceSizePerProblem - = solution.requiredHostSizeGroupedGemmSingle(problem,m_hardware); + = solution.requiredHostSizeGroupedGemmSingle(problem, *m_hardware); } return true; } @@ -124,7 +124,7 @@ namespace Tensile { for(int idx = 0; idx < problems->gemms.size(); idx++) { - size_t ws = solution.requiredWorkspaceSizeGroupedGemm(problems->gemms); + size_t ws = solution.requiredWorkspaceSizeGroupedGemm(problems->gemms, *m_hardware); auto& problem = problems->gemms[idx]; problem.setWorkspaceSizeGroupedGemm(ws); problem.setGroupedGemmCount(problems->gemms.size()); diff --git a/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp b/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp index f74b168bec..a63f32f459 100644 --- a/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp +++ b/tensilelite/Tensile/Source/lib/source/ContractionProblem.cpp @@ -1077,19 +1077,19 @@ namespace Tensile double gflop = 2 * problemSize * 1e-9; size_t aSize = 1; - for(size_t i = 0; i < m_a.dimensions(); ++i) + for(size_t i = 0; i < a().dimensions(); ++i) { - aSize *= m_a.sizes()[i]; + aSize *= a().sizes()[i]; } size_t bSize = 1; - for(size_t i = 0; i < m_b.dimensions(); ++i) + for(size_t i = 0; i < b().dimensions(); ++i) { - bSize *= m_b.sizes()[i]; + bSize *= b().sizes()[i]; } size_t cSize = 1; - for(size_t i = 0; i < m_c.dimensions(); ++i) + for(size_t i = 0; i < c().dimensions(); ++i) { - cSize *= m_c.sizes()[i]; + cSize *= c().sizes()[i]; } if(m_beta != 0) // If problem includes beta, update gflops and gbytes { @@ -1097,7 +1097,7 @@ namespace Tensile cSize *= 2; // Include read C and write D in gbytes } double gbyte - = (aSize * m_a.elementBytes() + bSize * m_b.elementBytes() + cSize * m_c.elementBytes()) + = (aSize * a().elementBytes() + bSize * b().elementBytes() + cSize * c().elementBytes()) * 1e-9; m_arithmeticIntensity = gflop / gbyte; From 9e8fa218f5628145e47f998bb993fcb0f1fadaac Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Tue, 23 Jul 2024 20:26:06 +0000 Subject: [PATCH 19/36] Fix race condition on flag reset --- tensilelite/Tensile/Components/StreamK.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index e5d4474f45..802520cd0f 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -297,6 +297,8 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele module.add(SCmpEQU32(src0=sgpr(tmpSgpr+2), src1=1, comment="check if ready")) module.add(SCBranchSCC0(labelName=skFixupLabel.getLabelName(), comment="if flag not set, wait and check again")) + # TODO Barrier here to sync all threads in workgroup, but maybe better to have separate flag for each wavefront (to be tested) + module.add(SBarrier(comment="wait for all workgroups before resetting flag")) module.add(SMovB32(dst=sgpr(tmpSgpr+2), src=0, comment="flag data")) module.add(SStoreB32(src=sgpr(tmpSgpr+2), base=sgpr("AddressFlags", 2), soffset=sgpr(tmpSgpr), smem=SMEMModifiers(glc=1), comment="reset flag")) writer.sgprPool.checkIn(tmpSgpr) From 5642e5ca1cefd4f50258c7309c5b5ae9647283ea Mon Sep 17 00:00:00 2001 From: "Aditya.Joshi" Date: Tue, 23 Jul 2024 22:40:50 +0000 Subject: [PATCH 20/36] Flag point to input synchronizer --- .../Tensile/Source/lib/source/ContractionSolution.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp index fa5b73c5bf..607ffb560d 100644 --- a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp +++ b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp @@ -725,12 +725,7 @@ namespace Tensile auto tiles = problem.getNumTiles(sizeMapping); size_t skGrid = getSKGrid(problem, *hardware, tiles); - // StreamK workspace + flags - args.template append("ws", inputs.ws); - void* ws = inputs.ws; - size_t flagsOffset = partialTileSize(skGrid); - void* flags = (void*)(static_cast(ws) + flagsOffset); - args.template append("Flags", flags); + args.template append("Flags", inputs.Synchronizer); } size_t startStrideCD = problemType.useInitialStridesCD ? 0 : 1; From 2f882d6f11f2d8a60222cddd6b518531f9ea59f2 Mon Sep 17 00:00:00 2001 From: "Aditya.Joshi" Date: Wed, 24 Jul 2024 14:24:50 +0000 Subject: [PATCH 21/36] point flag to synchronizer --- tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp index 607ffb560d..5c29054033 100644 --- a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp +++ b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp @@ -725,6 +725,8 @@ namespace Tensile auto tiles = problem.getNumTiles(sizeMapping); size_t skGrid = getSKGrid(problem, *hardware, tiles); + // StreamK workspace + flags + args.template append("ws", inputs.ws); args.template append("Flags", inputs.Synchronizer); } From 0c5159d470e9c1c3296c8c3f893415832f58c5e2 Mon Sep 17 00:00:00 2001 From: "Aditya.Joshi" Date: Wed, 24 Jul 2024 15:39:24 +0000 Subject: [PATCH 22/36] remove space allocated for flags --- .../Tensile/Source/lib/source/ContractionSolution.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp index 5c29054033..bd5889d31e 100644 --- a/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp +++ b/tensilelite/Tensile/Source/lib/source/ContractionSolution.cpp @@ -2752,12 +2752,6 @@ namespace Tensile size_t skGrid = getSKGrid(problem, hardware, tiles); // Get space required for partial tiles size += partialTileSize(skGrid); - // Add space for flags - // Flags for partial tiles - dword per flag for fast addressing and comparisons - // If tiles is evenly divided by grid size flags are not needed (DP mode) - if(tiles % skGrid != 0) - size += skGrid * 4; - // size *= batches; // TODO need tile and flag per batch } else { From 445e1f119dd72a654034052c519a6931214c1e9f Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Wed, 31 Jul 2024 22:43:30 +0000 Subject: [PATCH 23/36] Some bug fixes, general case sgemm tests passing --- tensilelite/Tensile/Components/StreamK.py | 6 +- tensilelite/Tensile/KernelWriter.py | 20 ++++ tensilelite/Tensile/KernelWriterAssembly.py | 122 +++++++++++--------- 3 files changed, 92 insertions(+), 56 deletions(-) diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index 802520cd0f..7362284115 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -593,8 +593,10 @@ def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, module.add(SWaitCnt(vmcnt=0, comment="wait for data store")) module.add(SBarrier(comment="store all data before setting flag")) module.add(SLShiftLeftB32(dst=sgpr(tmpSgpr), src=sgpr("StreamKIdx"), shiftHex=log2(4), comment="flag offset based on CTA index")) - module.add(SMovB32(dst=sgpr(tmpSgpr+2), src=1, comment="flag data")) - module.add(SStoreB32(src=sgpr(tmpSgpr+2), base=sgpr("AddressFlags", 2), soffset=sgpr(tmpSgpr), smem=SMEMModifiers(glc=1), comment="set flag")) + with writer.allocTmpSgpr(1) as flagSgprRes: + flagSgpr = flagSgprRes.idx + module.add(SMovB32(dst=sgpr(flagSgpr), src=1, comment="flag data")) + module.add(SStoreB32(src=sgpr(flagSgpr), base=sgpr("AddressFlags", 2), soffset=sgpr(tmpSgpr), smem=SMEMModifiers(glc=1), comment="set flag")) module.add(SWaitCnt(lgkmcnt=0, comment="wait for flag")) # TODO just for testing # TODO - if this is the last tile, don't need to jump to next instruction diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 99a8c1ed10..505c56308d 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -1359,6 +1359,10 @@ def setupNewTile(self, kernel, tensorParametersA, tensorParametersB, isOptNLL=Fa tPM = tensorParametersB["tpsMetadata"] tPMRef = tensorParametersB + if kernel["StreamK"] != 0: + module.add(self.localReadAddresses(kernel, tensorParametersA, tensorParametersB, tPM)) + module.add(self.localWriteAddresses(kernel, tensorParametersA, tensorParametersB, tPM)) + # tile assignments module.addComment1("global read addresses: tile offset assignment a") module.add(self.graTileAssignment(kernel, tensorParametersA)) @@ -3672,6 +3676,8 @@ def readWriteVectors(mat, vw, kernel): self.states.nonPostLoopSgpr.remove("AddressB") self.states.nonPostLoopSgpr.remove("StridesA") self.states.nonPostLoopSgpr.remove("StridesB") + self.states.nonPostLoopSgpr.remove("NumWorkGroups0") + self.states.nonPostLoopSgpr.remove("NumWorkGroups1") self.states.preloadScaleA = False self.states.preloadScaleB = False @@ -3911,6 +3917,20 @@ def readWriteVectors(mat, vw, kernel): def functionSignature(self): return "" + ############################################################################## + # Local Read Addresses + ############################################################################## + @abc.abstractmethod + def localReadAddresses(self, kernel, tPA, tPB, tPM): + return "" + + ############################################################################## + # Local Write Addresses + ############################################################################## + @abc.abstractmethod + def localWriteAddresses(self, kernel, tPA, tPB, tPM): + return "" + ############################################################################## # Allocate Resources ############################################################################## diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index d91510269e..f3af7a613f 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -1116,6 +1116,69 @@ def getKernelArgLoadModule(self, kernel, sgprStartIdx, numsOfLoad, preloadNum): sgprOffset += (self.states.rpga * self.states.bpr) return kernelArgs + def localReadAddresses(self, kernel, tPA, tPB, tPM): + module = Module("Local Read Addresses") + + #################################### + # Local Read Addresses + #################################### + module.addComment2("Local Read Addresses") + + # tile assignments + module.addComment1("local read addresses: tile assignments a/b") + module.add(self.lraTileAssignment(kernel, tPA, tPB)) + + # final offsets + module.addComment1("local read addresses: final offsets a") + module.add(self.lraFinalOffset(kernel, tPA)) + if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: + module.addComment1("local read addresses: final offsets metadata") + module.add(self.lraFinalOffset(kernel, tPM)) + module.addComment1("local read addresses: final offsets b") + module.add(self.lraFinalOffset(kernel, tPB)) + + # declare addresses + module.addComment1("local read addresses: declare addresses a") + module.add(self.lraDeclareAddresses(kernel, tPA)) + if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: + module.addComment1("local read addresses: declare addresses metadata") + module.add(self.lraDeclareAddresses(kernel, tPM)) + module.addComment1("local read addresses: declare addresses b") + module.add(self.lraDeclareAddresses(kernel, tPB)) + + return module + + def localWriteAddresses(self, kernel, tPA, tPB, tPM): + module = Module("Local Write Addresses") + + #################################### + # Local Write Addresses + #################################### + module.addComment2("Local Write Addresses") + + # tile assignments + module.add(self.lwaTileAssignment(kernel, tPA)) + if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: + module.add(self.lwaTileAssignment(kernel, tPM)) + module.add(self.lwaTileAssignment(kernel, tPB)) + + # unroll assignments + module.add(self.lwaUnrollAssignment(kernel, tPA)) + if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: + module.add(self.lwaUnrollAssignment(kernel, tPM)) + module.add(self.lwaUnrollAssignment(kernel, tPB)) + + # first offsets + module.addComment1("local write addresses: first offset a") + module.add(self.lwaFirstOffset(kernel, tPA)) + if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: + module.addComment1("local write addresses: first offset metadata") + module.add(self.lwaFirstOffset(kernel, tPM)) + module.addComment1("local write addresses: first offset b") + module.add(self.lwaFirstOffset(kernel, tPB)) + + return module + def defineAndResources(self, kernel, tPA, tPB, tPM): module = Module("allocateResources") module.add(self.macroAndSet(kernel, tPA, tPB)) @@ -1303,9 +1366,7 @@ def defineAndResources(self, kernel, tPA, tPB, tPM): moduleScaleAB.add(label) moduleWg = Module("Calculate Workgroup") - #################################### - # Local Read Addresses - #################################### + # C regs are not used during initialization so mark them as available - # we will claim then just before the start of the unroll loop: self.vgprPool.add(self.states.a.startVgprValu , \ @@ -1323,57 +1384,10 @@ def defineAndResources(self, kernel, tPA, tPB, tPM): moduleWg.addComment0("init: add agpr [%u...%u) to pool" % \ (0, numAccvgprs)) - moduleWg.addComment2("Local Read Addresses") - - # tile assignments - moduleWg.addComment1("local read addresses: tile assignments a/b") - moduleWg.add(self.lraTileAssignment(kernel, tPA, tPB)) - - # final offsets - moduleWg.addComment1("local read addresses: final offsets a") - moduleWg.add(self.lraFinalOffset(kernel, tPA)) - if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: - moduleWg.addComment1("local read addresses: final offsets metadata") - moduleWg.add(self.lraFinalOffset(kernel, tPM)) - moduleWg.addComment1("local read addresses: final offsets b") - moduleWg.add(self.lraFinalOffset(kernel, tPB)) - - # declare addresses - moduleWg.addComment1("local read addresses: declare addresses a") - moduleWg.add(self.lraDeclareAddresses(kernel, tPA)) - if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: - moduleWg.addComment1("local read addresses: declare addresses metadata") - moduleWg.add(self.lraDeclareAddresses(kernel, tPM)) - moduleWg.addComment1("local read addresses: declare addresses b") - moduleWg.add(self.lraDeclareAddresses(kernel, tPB)) - - - #################################### - # Local Write Addresses - #################################### - moduleWg.addComment2("Local Write Addresses") - - # tile assignments - moduleWg.add(self.lwaTileAssignment(kernel, tPA)) - if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: - moduleWg.add(self.lwaTileAssignment(kernel, tPM)) - moduleWg.add(self.lwaTileAssignment(kernel, tPB)) - - # unroll assignments - moduleWg.add(self.lwaUnrollAssignment(kernel, tPA)) - if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: - moduleWg.add(self.lwaUnrollAssignment(kernel, tPM)) - moduleWg.add(self.lwaUnrollAssignment(kernel, tPB)) - - # first offsets - moduleWg.addComment1("local write addresses: first offset a") - moduleWg.add(self.lwaFirstOffset(kernel, tPA)) - if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: - moduleWg.addComment1("local write addresses: first offset metadata") - moduleWg.add(self.lwaFirstOffset(kernel, tPM)) - moduleWg.addComment1("local write addresses: first offset b") - moduleWg.add(self.lwaFirstOffset(kernel, tPB)) - + if kernel["StreamK"] == 0: + moduleWg.add(self.localReadAddresses(kernel, tPA, tPB, tPM)) + moduleWg.add(self.localWriteAddresses(kernel, tPA, tPB, tPM)) + def waitForArgsToLoad(): if kernel["ProblemType"]["SupportUserArgs"]: moduleWg.add(SWaitCnt(lgkmcnt=0, comment="wait for %u/%u bytes of kern args" % \ From f7348367fc9bbab1e66200636b2da86ae66b76c0 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Fri, 16 Aug 2024 15:12:45 +0000 Subject: [PATCH 24/36] SOme bug fixes --- tensilelite/Tensile/Components/StreamK.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index 7362284115..140bbf77e0 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -24,7 +24,7 @@ SCmpLtU32, SCSelectB32, sMagicDivAlg2, SMulI32, SSubU32, SMinU32, SMovB32, SCBranchSCC1, SCmpLeU32, VMovB32, vgpr, \ SAddCU32, SCmpGtU32, SCMovB32, SAddI32, SCmpEQU32, SCBranchSCC0, SLShiftLeftB32, SLoadB32, SWaitCnt, SMEMModifiers, \ log2, SBarrier, SStoreB32, SLongBranchPositive, SBranch, ceilDivide, replaceHolder, SNop, staticMultiply, SSleep, \ - VAddF32 + VAddF32, VAddF64 from ..Common import print2 # from ..TensileInstructions.Containers import SMEMModifiers from ..Component import Component @@ -268,7 +268,8 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele # if we did not start the tile, store partials # branch to beta == 0 store path module.add(SCmpEQU32(src0=sgpr("StreamKLocalStart"), src1=0, comment="does wg start tile?")) - module.add(SCBranchSCC0(labelName=skPartialsLabel.getLabelName(), comment="Branch if not start tile, store partials")) + module.add(writer.longBranchScc0(skPartialsLabel, posNeg=1)) + # module.add(SCBranchSCC0(labelName=skPartialsLabel.getLabelName(), comment="Branch if not start tile, store partials")) if kernel["DebugStreamK"] & 1 == 0: # if we started and finished the tile, regular store code @@ -1060,7 +1061,7 @@ def fixupStep(self, writer, kernel, vectorWidths, elements, edges, tmpVgpr, cvtV def fixupBatch(self, writer, kernel, ss, batchIdx, edge, gwvw, \ batchElements, addrD, addrC, \ - tmpVgpr, tmpCVTVgpr, batchElementSgprs, tmpSgpr, codeAccVgprRead, codeAccVgprWrite): + tmpVgpr, cvtVgprStruct, batchElementSgprs, tmpSgpr, codeAccVgprRead, codeAccVgprWrite): module = Module("StreamK Common fixupBatch") module.addComment0("optSingleColVgpr=%u optSharedColVgpr=%u optSGPRUsage=%s optSrdIncForRow=%u" % \ From 593e98f0bb2c367bcde281144146f3771d7b2d46 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Wed, 21 Aug 2024 16:09:50 +0000 Subject: [PATCH 25/36] Merge fixes for stream-k --- tensilelite/Tensile/Components/StreamK.py | 4 ++-- tensilelite/Tensile/KernelWriterAssembly.py | 8 ++++++-- tensilelite/Tensile/Source/lib/source/Debug.cpp | 8 ++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index 15e7482793..184d4978b9 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -639,7 +639,7 @@ def partialsWriteBatch(self, writer, kernel, ss, batchIdx, applyAlpha, beta, edg # allow expanding vgpr pool for OptNLL # preventOverflow = (not isOptNLL) # ss.setupStoreElementsForBatch(kernel, gwvw, batchElements, batchElementSgprs, isOptNLL=isOptNLL, isWorkspace=True) - ss.setupStoreElementsForBatch(kernel, gwvw, batchElements, batchElementSgprs, isOptNLL=False, biasDim=0, isWorkspace=True) + ss.setupStoreElementsForBatch(kernel, gwvw, batchElements, batchElementSgprs, isOptNLL=False, factorDim=0, isWorkspace=True) storesIssued = 0 tmpS01 = tmpSgpr # scratch sgprs @@ -1087,7 +1087,7 @@ def fixupBatch(self, writer, kernel, ss, batchIdx, edge, gwvw, \ # allow expanding vgpr pool for OptNLL # preventOverflow = True #(not isOptNLL) # ss.setupStoreElementsForBatch(kernel, gwvw, batchElements, batchElementSgprs, preventOverflow=preventOverflow, isWorkspace=True) - ss.setupStoreElementsForBatch(kernel, gwvw, batchElements, batchElementSgprs, isOptNLL=False, biasDim=0, isWorkspace=True) + ss.setupStoreElementsForBatch(kernel, gwvw, batchElements, batchElementSgprs, isOptNLL=False, factorDim=0, isWorkspace=True) loadsIssued = 0 storesIssued = 0 diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 14f10d2c25..e9bed4cb36 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -1204,6 +1204,8 @@ def localWriteAddresses(self, kernel, tPA, tPB, tPM): module.addComment1("local write addresses: first offset b") module.add(self.lwaFirstOffset(kernel, tPB)) + return module + def wgmXCC(self, kernel, tmpSgprNumWorkGroups): module = Module("WGMXCC") module.addComment1("remap workgroup to XCCs") @@ -1559,7 +1561,8 @@ def calculateWG(): sgprArgType = None # Cannot be used after this point module.add(SCBranchSCC0(labelName=labelMultiGemm.getLabelName())) module.add(fastdeepcopy(moduleWg)) - module.add(self.remapWgSerial(kernel, earlyStop=False)) + if kernel["StreamK"] == 0: + module.add(self.remapWgSerial(kernel, earlyStop=False)) module.add(SBranch(labelName=labelMultiGemmEnd.getLabelName())) module.add(labelMultiGemm) @@ -1769,7 +1772,8 @@ def calculateWG(): earlyReturnModule.add(SEndpgm()) earlyReturnModule.add(noEarlyReturnLabel) module.add(earlyReturnModule) - module.add(self.remapWgSerial(kernel)) + if kernel["StreamK"] == 0: + module.add(self.remapWgSerial(kernel)) module.addSpaceLine() module.add(labelMultiGemmEnd) diff --git a/tensilelite/Tensile/Source/lib/source/Debug.cpp b/tensilelite/Tensile/Source/lib/source/Debug.cpp index 38f741955c..d7e909ae34 100644 --- a/tensilelite/Tensile/Source/lib/source/Debug.cpp +++ b/tensilelite/Tensile/Source/lib/source/Debug.cpp @@ -155,6 +155,10 @@ namespace Tensile return m_gridbasedKdTree; } + bool Debug::gridBasedBatchExp() const + { + return m_gridbasedBatchExp; + } Debug::Debug() : m_value(DEBUG_SM) @@ -199,6 +203,10 @@ namespace Tensile const char* tensile_gridbased_kdtree = std::getenv("TENSILE_GRIDBASED_KDTREE"); if(tensile_gridbased_kdtree) m_gridbasedKdTree = strtol(tensile_gridbased_kdtree, nullptr, 0) != 0; + + const char* tensile_gridbased_batch_exp = std::getenv("TENSILE_GRIDBASED_BATCH_EXP"); + if(tensile_gridbased_batch_exp) + m_gridbasedBatchExp = strtol(tensile_gridbased_batch_exp, nullptr, 0) != 0; } } // namespace Tensile From 3a96ecf96c07dd097292723ff3432781cf80cf7f Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Wed, 21 Aug 2024 16:10:38 +0000 Subject: [PATCH 26/36] Some stream-k test files --- .../Tests/common/streamk/sk_bgemm_quick.yaml | 805 ++++++++++++++++++ .../Tests/common/streamk/sk_hgemm_quick.yaml | 805 ++++++++++++++++++ .../Tests/common/streamk/sk_sgemm_quick.yaml | 793 +++++++++++++++++ 3 files changed, 2403 insertions(+) create mode 100644 tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml create mode 100644 tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml create mode 100644 tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml diff --git a/tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml b/tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml new file mode 100644 index 0000000000..daf5843091 --- /dev/null +++ b/tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml @@ -0,0 +1,805 @@ + +GlobalParameters: + NumElementsToValidate: -1 + BoundsCheck: False + KernelTime: False + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + DataInitTypeA: 12 + DataInitTypeB: 13 + DataInitTypeC: 12 + # DataInitTypeC: 1 + # ValidationPrintValids: True + MaxWorkspaceSize: 134217728 + # PrintSolutionRejectionReason: True + # ForceGenerateKernel: True + # GenerateSourcesAndExit: True + NumWarmups: 0 + EnqueuesPerSync: 1 + # NumBenchmarks: 10 + +BenchmarkProblems: + + - # HGEMM NT + - # ProblemType + OperationType: GEMM + DataType: b + DestDataType: b + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: False + TransposeB: True + UseBeta: True + Batched: True + + - # HGEMM NT - Test MatrixInstruction variants + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 64 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [8] + - GlobalReadVectorWidthB: [8] + - GlobalSplitU: [0] + # - LocalReadVectorWidth: [8] + - MatrixInstruction: + - [32, 32, 8, 1, 1, 4,4, 2,2] + - [32, 32, 8, 1, 1, 3,3, 2,2] + - [32, 32, 8, 1, 1, 2,2, 2,2] + - [32, 32, 8, 1, 1, 1,1, 2,2] + - [16, 16, 16, 1, 1, 8,8, 2,2] + - [16, 16, 16, 1, 1, 8,4, 2,2] + - [16, 16, 16, 1, 1, 8,2, 2,2] + - [16, 16, 16, 1, 1, 8,1, 2,2] + - [16, 16, 16, 1, 1, 4,8, 2,2] + - [16, 16, 16, 1, 1, 4,4, 2,2] + - [16, 16, 16, 1, 1, 4,2, 2,2] + - [16, 16, 16, 1, 1, 4,1, 2,2] + - [16, 16, 16, 1, 1, 2,8, 2,2] + - [16, 16, 16, 1, 1, 2,4, 2,2] + - [16, 16, 16, 1, 1, 2,2, 2,2] + - [16, 16, 16, 1, 1, 2,1, 2,2] + - [16, 16, 16, 1, 1, 1,8, 2,2] + - [16, 16, 16, 1, 1, 1,4, 2,2] + - [16, 16, 16, 1, 1, 1,2, 2,2] + - [16, 16, 16, 1, 1, 1,1, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + - StreamK: [3] + - TransposeLDS: [0] + # - VectorWidthA: [4] + # - VectorWidthB: [4] + - WorkGroupMapping: [1] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + - Exact: [4103, 4103, 1, 1031] + + - # HGEMM NT - Test DepthU, WGM, VW + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 32, 64 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [2, 4, 8] + - GlobalReadVectorWidthB: [2, 4, 8] + - GlobalSplitU: [0] + # - LocalReadVectorWidth: [8] + - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + - [16, 16, 16, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + - StreamK: [3] + - TransposeLDS: [0] + # - VectorWidthA: [1, 2, 4] + # - VectorWidthB: [1, 2, 4] + - WorkGroupMapping: [0, 1, 2, 4, 8, 16, 32, 64] # works + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + - Exact: [2055, 2055, 1, 1031] + + - # HGEMM NT - Test tuning params + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [0, 1] + - DepthU: [64] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [8] + - GlobalReadVectorWidthB: [8] + - GlobalSplitU: [0] + # - LocalReadVectorWidth: [1] + - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + - [16, 16, 16, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - StaggerU: [0, 32] + - ScheduleIterAlg: [3] + - SourceSwap: [False, True] + - StoreRemapVectorWidth: [0, 4] + # - StoreVectorWidth: [1] + - StreamK: [3] + - PrefetchLocalRead: [1, 3] + - NonTemporalC: [0, 7] + - NonTemporalD: [0, 7] + - TransposeLDS: [0] + # - VectorWidthA: [1] + # - VectorWidthB: [1] + - WorkGroupMapping: [1] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + - Exact: [2055, 2055, 1, 1031] + + # - # HGEMM NT - Test size range + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [2, 4, 8] + # - GlobalReadVectorWidthB: [2, 4, 8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1, 2, 4] + # # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [1, 2, 4] + # # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Range: [ [32, 249, 2048], [32, 249, 2048], [1], [32, 249, 2048] ] + + - # HGEMM NN + - # ProblemType + OperationType: GEMM + DataType: b + DestDataType: b + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: False + TransposeB: False + UseBeta: True + Batched: True + + - # HGEMM NN - Test MatrixInstruction variants + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 64 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [8] + - GlobalReadVectorWidthB: [8] + - GlobalSplitU: [0] + # - LocalReadVectorWidth: [8] + - MatrixInstruction: + - [32, 32, 8, 1, 1, 4,4, 2,2] + - [32, 32, 8, 1, 1, 3,3, 2,2] + - [32, 32, 8, 1, 1, 2,2, 2,2] + - [32, 32, 8, 1, 1, 1,1, 2,2] + - [16, 16, 16, 1, 1, 8,8, 2,2] + - [16, 16, 16, 1, 1, 8,4, 2,2] + - [16, 16, 16, 1, 1, 8,2, 2,2] + - [16, 16, 16, 1, 1, 8,1, 2,2] + - [16, 16, 16, 1, 1, 4,8, 2,2] + - [16, 16, 16, 1, 1, 4,4, 2,2] + - [16, 16, 16, 1, 1, 4,2, 2,2] + - [16, 16, 16, 1, 1, 4,1, 2,2] + - [16, 16, 16, 1, 1, 2,8, 2,2] + - [16, 16, 16, 1, 1, 2,4, 2,2] + - [16, 16, 16, 1, 1, 2,2, 2,2] + - [16, 16, 16, 1, 1, 2,1, 2,2] + - [16, 16, 16, 1, 1, 1,8, 2,2] + - [16, 16, 16, 1, 1, 1,4, 2,2] + - [16, 16, 16, 1, 1, 1,2, 2,2] + - [16, 16, 16, 1, 1, 1,1, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + - StreamK: [3] + - TransposeLDS: [0] + # - VectorWidthA: [4] + # - VectorWidthB: [4] + - WorkGroupMapping: [1] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + - Exact: [4103, 4103, 1, 1031] + + # - # HGEMM NN - Test DepthU, WGM, VW + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [ 32, 64 ] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [2, 4, 8] + # - GlobalReadVectorWidthB: [2, 4, 8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [8] + # - MatrixInstruction: + # # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [1, 2, 4] + # # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [0, 1, 2, 4, 8, 16, 32, 64] # works + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # # - Exact: [4103, 4103, 1, 1031] + # - Exact: [2055, 2055, 1, 1031] + + # - # HGEMM NN - Test tuning params + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [0, 1] + # - DepthU: [64] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [8] + # - GlobalReadVectorWidthB: [8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - StaggerU: [0, 32] + # - ScheduleIterAlg: [3] + # - SourceSwap: [False, True] + # - StoreRemapVectorWidth: [0, 4] + # # - StoreVectorWidth: [1] + # - StreamK: [3] + # - PrefetchLocalRead: [1, 3] + # - NonTemporalC: [0, 7] + # - NonTemporalD: [0, 7] + # - TransposeLDS: [0, 1] + # # - VectorWidthA: [1] + # # - VectorWidthB: [1] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # # - Exact: [4103, 4103, 1, 1031] + # - Exact: [2055, 2055, 1, 1031] + + # - # HGEMM NN - Test size range + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [2, 4, 8] + # - GlobalReadVectorWidthB: [2, 4, 8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1, 2, 4] + # # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [1, 2, 4] + # # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Range: [ [32, 249, 2048], [32, 249, 2048], [1], [32, 249, 2048] ] + + - # HGEMM TN + - # ProblemType + OperationType: GEMM + DataType: b + DestDataType: b + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + + # - # HGEMM TN - Test MatrixInstruction variants + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [ 64 ] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [8] + # - GlobalReadVectorWidthB: [8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [8] + # - MatrixInstruction: + # - [32, 32, 8, 1, 1, 4,4, 2,2] + # - [32, 32, 8, 1, 1, 3,3, 2,2] + # - [32, 32, 8, 1, 1, 2,2, 2,2] + # - [32, 32, 8, 1, 1, 1,1, 2,2] + # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 8,4, 2,2] + # - [16, 16, 16, 1, 1, 8,2, 2,2] + # - [16, 16, 16, 1, 1, 8,1, 2,2] + # - [16, 16, 16, 1, 1, 4,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - [16, 16, 16, 1, 1, 4,2, 2,2] + # - [16, 16, 16, 1, 1, 4,1, 2,2] + # - [16, 16, 16, 1, 1, 2,8, 2,2] + # - [16, 16, 16, 1, 1, 2,4, 2,2] + # - [16, 16, 16, 1, 1, 2,2, 2,2] + # - [16, 16, 16, 1, 1, 2,1, 2,2] + # - [16, 16, 16, 1, 1, 1,8, 2,2] + # - [16, 16, 16, 1, 1, 1,4, 2,2] + # - [16, 16, 16, 1, 1, 1,2, 2,2] + # - [16, 16, 16, 1, 1, 1,1, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [4] + # # - VectorWidthB: [4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + + - # HGEMM TN - Test DepthU, WGM, VW + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 32, 64 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [2, 4, 8] + - GlobalReadVectorWidthB: [2, 4, 8] + - GlobalSplitU: [0] + # - LocalReadVectorWidth: [8] + - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + - [16, 16, 16, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + - StreamK: [3] + - TransposeLDS: [0] + # - VectorWidthA: [1, 2, 4] + # - VectorWidthB: [1, 2, 4] + - WorkGroupMapping: [0, 1, 2, 4, 8, 16, 32, 64] # works + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + - Exact: [2055, 2055, 1, 1031] + + # - # HGEMM TN - Test tuning params + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [0, 1] + # - DepthU: [64] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [8] + # - GlobalReadVectorWidthB: [8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - StaggerU: [0, 32] + # - ScheduleIterAlg: [3] + # - SourceSwap: [False, True] + # - StoreRemapVectorWidth: [0, 4] + # # - StoreVectorWidth: [1] + # - StreamK: [3] + # - PrefetchLocalRead: [1, 3] + # - NonTemporalC: [0, 7] + # - NonTemporalD: [0, 7] + # - TransposeLDS: [0, 1] + # # - VectorWidthA: [1] + # # - VectorWidthB: [1] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # # - Exact: [4103, 4103, 1, 1031] + # - Exact: [2055, 2055, 1, 1031] + + # - # HGEMM TN - Test size range + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [2, 4, 8] + # - GlobalReadVectorWidthB: [2, 4, 8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1, 2, 4] + # # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [1, 2, 4] + # # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Range: [ [32, 249, 2048], [32, 249, 2048], [1], [32, 249, 2048] ] + + - # HGEMM TT + - # ProblemType + OperationType: GEMM + DataType: b + DestDataType: b + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: True + UseBeta: True + Batched: True + + # - # HGEMM TT - Test MatrixInstruction variants + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [ 64 ] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [8] + # - GlobalReadVectorWidthB: [8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [8] + # - MatrixInstruction: + # - [32, 32, 8, 1, 1, 4,4, 2,2] + # - [32, 32, 8, 1, 1, 3,3, 2,2] + # - [32, 32, 8, 1, 1, 2,2, 2,2] + # - [32, 32, 8, 1, 1, 1,1, 2,2] + # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 8,4, 2,2] + # - [16, 16, 16, 1, 1, 8,2, 2,2] + # - [16, 16, 16, 1, 1, 8,1, 2,2] + # - [16, 16, 16, 1, 1, 4,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - [16, 16, 16, 1, 1, 4,2, 2,2] + # - [16, 16, 16, 1, 1, 4,1, 2,2] + # - [16, 16, 16, 1, 1, 2,8, 2,2] + # - [16, 16, 16, 1, 1, 2,4, 2,2] + # - [16, 16, 16, 1, 1, 2,2, 2,2] + # - [16, 16, 16, 1, 1, 2,1, 2,2] + # - [16, 16, 16, 1, 1, 1,8, 2,2] + # - [16, 16, 16, 1, 1, 1,4, 2,2] + # - [16, 16, 16, 1, 1, 1,2, 2,2] + # - [16, 16, 16, 1, 1, 1,1, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [4] + # # - VectorWidthB: [4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + + # - # HGEMM TT - Test DepthU, WGM, VW + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [ 32, 64 ] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [2, 4, 8] + # - GlobalReadVectorWidthB: [2, 4, 8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [8] + # - MatrixInstruction: + # # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [1, 2, 4] + # # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [0, 1, 2, 4, 8, 16, 32, 64] # works + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # # - Exact: [4103, 4103, 1, 1031] + # - Exact: [2055, 2055, 1, 1031] + + - # HGEMM TT - Test tuning params + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [0, 1] + - DepthU: [64] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [8] + - GlobalReadVectorWidthB: [8] + - GlobalSplitU: [0] + # - LocalReadVectorWidth: [1] + - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + - [16, 16, 16, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - StaggerU: [0, 32] + - ScheduleIterAlg: [3] + - SourceSwap: [False, True] + - StoreRemapVectorWidth: [0, 4] + # - StoreVectorWidth: [1] + - StreamK: [3] + - PrefetchLocalRead: [1, 3] + - NonTemporalC: [0, 7] + - NonTemporalD: [0, 7] + - TransposeLDS: [0, 1] + # - VectorWidthA: [1] + # - VectorWidthB: [1] + - WorkGroupMapping: [1] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + - Exact: [2055, 2055, 1, 1031] + + # - # HGEMM TT - Test size range + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [2, 4, 8] + # - GlobalReadVectorWidthB: [2, 4, 8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1, 2, 4] + # # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [1, 2, 4] + # # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Range: [ [32, 249, 2048], [32, 249, 2048], [1], [32, 249, 2048] ] diff --git a/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml b/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml new file mode 100644 index 0000000000..72525a7919 --- /dev/null +++ b/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml @@ -0,0 +1,805 @@ + +GlobalParameters: + NumElementsToValidate: -1 + BoundsCheck: False + KernelTime: False + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + DataInitTypeA: 12 + DataInitTypeB: 13 + DataInitTypeC: 12 + # DataInitTypeC: 1 + # ValidationPrintValids: True + MaxWorkspaceSize: 134217728 + # PrintSolutionRejectionReason: True + # ForceGenerateKernel: True + # GenerateSourcesAndExit: True + NumWarmups: 0 + EnqueuesPerSync: 1 + # NumBenchmarks: 10 + +BenchmarkProblems: + + - # HGEMM NT + - # ProblemType + OperationType: GEMM + DataType: h + DestDataType: h + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: False + TransposeB: True + UseBeta: True + Batched: True + + - # HGEMM NT - Test MatrixInstruction variants + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 64 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [8] + - GlobalReadVectorWidthB: [8] + - GlobalSplitU: [0] + # - LocalReadVectorWidth: [8] + - MatrixInstruction: + - [32, 32, 8, 1, 1, 4,4, 2,2] + - [32, 32, 8, 1, 1, 3,3, 2,2] + - [32, 32, 8, 1, 1, 2,2, 2,2] + - [32, 32, 8, 1, 1, 1,1, 2,2] + - [16, 16, 16, 1, 1, 8,8, 2,2] + - [16, 16, 16, 1, 1, 8,4, 2,2] + - [16, 16, 16, 1, 1, 8,2, 2,2] + - [16, 16, 16, 1, 1, 8,1, 2,2] + - [16, 16, 16, 1, 1, 4,8, 2,2] + - [16, 16, 16, 1, 1, 4,4, 2,2] + - [16, 16, 16, 1, 1, 4,2, 2,2] + - [16, 16, 16, 1, 1, 4,1, 2,2] + - [16, 16, 16, 1, 1, 2,8, 2,2] + - [16, 16, 16, 1, 1, 2,4, 2,2] + - [16, 16, 16, 1, 1, 2,2, 2,2] + - [16, 16, 16, 1, 1, 2,1, 2,2] + - [16, 16, 16, 1, 1, 1,8, 2,2] + - [16, 16, 16, 1, 1, 1,4, 2,2] + - [16, 16, 16, 1, 1, 1,2, 2,2] + - [16, 16, 16, 1, 1, 1,1, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + - StreamK: [3] + - TransposeLDS: [0] + # - VectorWidthA: [4] + # - VectorWidthB: [4] + - WorkGroupMapping: [1] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + - Exact: [4103, 4103, 1, 1031] + + - # HGEMM NT - Test DepthU, WGM, VW + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 32, 64 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [2, 4, 8] + - GlobalReadVectorWidthB: [2, 4, 8] + - GlobalSplitU: [0] + # - LocalReadVectorWidth: [8] + - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + - [16, 16, 16, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + - StreamK: [3] + - TransposeLDS: [0] + # - VectorWidthA: [1, 2, 4] + # - VectorWidthB: [1, 2, 4] + - WorkGroupMapping: [0, 1, 2, 4, 8, 16, 32, 64] # works + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + - Exact: [2055, 2055, 1, 1031] + + - # HGEMM NT - Test tuning params + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [0, 1] + - DepthU: [64] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [8] + - GlobalReadVectorWidthB: [8] + - GlobalSplitU: [0] + # - LocalReadVectorWidth: [1] + - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + - [16, 16, 16, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - StaggerU: [0, 32] + - ScheduleIterAlg: [3] + - SourceSwap: [False, True] + - StoreRemapVectorWidth: [0, 4] + # - StoreVectorWidth: [1] + - StreamK: [3] + - PrefetchLocalRead: [1, 3] + - NonTemporalC: [0, 7] + - NonTemporalD: [0, 7] + - TransposeLDS: [0] + # - VectorWidthA: [1] + # - VectorWidthB: [1] + - WorkGroupMapping: [1] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + - Exact: [2055, 2055, 1, 1031] + + # - # HGEMM NT - Test size range + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [2, 4, 8] + # - GlobalReadVectorWidthB: [2, 4, 8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1, 2, 4] + # # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [1, 2, 4] + # # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Range: [ [32, 249, 2048], [32, 249, 2048], [1], [32, 249, 2048] ] + + - # HGEMM NN + - # ProblemType + OperationType: GEMM + DataType: h + DestDataType: h + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: False + TransposeB: False + UseBeta: True + Batched: True + + - # HGEMM NN - Test MatrixInstruction variants + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 64 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [8] + - GlobalReadVectorWidthB: [8] + - GlobalSplitU: [0] + # - LocalReadVectorWidth: [8] + - MatrixInstruction: + - [32, 32, 8, 1, 1, 4,4, 2,2] + - [32, 32, 8, 1, 1, 3,3, 2,2] + - [32, 32, 8, 1, 1, 2,2, 2,2] + - [32, 32, 8, 1, 1, 1,1, 2,2] + - [16, 16, 16, 1, 1, 8,8, 2,2] + - [16, 16, 16, 1, 1, 8,4, 2,2] + - [16, 16, 16, 1, 1, 8,2, 2,2] + - [16, 16, 16, 1, 1, 8,1, 2,2] + - [16, 16, 16, 1, 1, 4,8, 2,2] + - [16, 16, 16, 1, 1, 4,4, 2,2] + - [16, 16, 16, 1, 1, 4,2, 2,2] + - [16, 16, 16, 1, 1, 4,1, 2,2] + - [16, 16, 16, 1, 1, 2,8, 2,2] + - [16, 16, 16, 1, 1, 2,4, 2,2] + - [16, 16, 16, 1, 1, 2,2, 2,2] + - [16, 16, 16, 1, 1, 2,1, 2,2] + - [16, 16, 16, 1, 1, 1,8, 2,2] + - [16, 16, 16, 1, 1, 1,4, 2,2] + - [16, 16, 16, 1, 1, 1,2, 2,2] + - [16, 16, 16, 1, 1, 1,1, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + - StreamK: [3] + - TransposeLDS: [0] + # - VectorWidthA: [4] + # - VectorWidthB: [4] + - WorkGroupMapping: [1] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + - Exact: [4103, 4103, 1, 1031] + + # - # HGEMM NN - Test DepthU, WGM, VW + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [ 32, 64 ] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [2, 4, 8] + # - GlobalReadVectorWidthB: [2, 4, 8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [8] + # - MatrixInstruction: + # # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [1, 2, 4] + # # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [0, 1, 2, 4, 8, 16, 32, 64] # works + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # # - Exact: [4103, 4103, 1, 1031] + # - Exact: [2055, 2055, 1, 1031] + + # - # HGEMM NN - Test tuning params + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [0, 1] + # - DepthU: [64] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [8] + # - GlobalReadVectorWidthB: [8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - StaggerU: [0, 32] + # - ScheduleIterAlg: [3] + # - SourceSwap: [False, True] + # - StoreRemapVectorWidth: [0, 4] + # # - StoreVectorWidth: [1] + # - StreamK: [3] + # - PrefetchLocalRead: [1, 3] + # - NonTemporalC: [0, 7] + # - NonTemporalD: [0, 7] + # - TransposeLDS: [0, 1] + # # - VectorWidthA: [1] + # # - VectorWidthB: [1] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # # - Exact: [4103, 4103, 1, 1031] + # - Exact: [2055, 2055, 1, 1031] + + # - # HGEMM NN - Test size range + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [2, 4, 8] + # - GlobalReadVectorWidthB: [2, 4, 8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1, 2, 4] + # # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [1, 2, 4] + # # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Range: [ [32, 249, 2048], [32, 249, 2048], [1], [32, 249, 2048] ] + + - # HGEMM TN + - # ProblemType + OperationType: GEMM + DataType: h + DestDataType: h + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + + # - # HGEMM TN - Test MatrixInstruction variants + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [ 64 ] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [8] + # - GlobalReadVectorWidthB: [8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [8] + # - MatrixInstruction: + # - [32, 32, 8, 1, 1, 4,4, 2,2] + # - [32, 32, 8, 1, 1, 3,3, 2,2] + # - [32, 32, 8, 1, 1, 2,2, 2,2] + # - [32, 32, 8, 1, 1, 1,1, 2,2] + # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 8,4, 2,2] + # - [16, 16, 16, 1, 1, 8,2, 2,2] + # - [16, 16, 16, 1, 1, 8,1, 2,2] + # - [16, 16, 16, 1, 1, 4,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - [16, 16, 16, 1, 1, 4,2, 2,2] + # - [16, 16, 16, 1, 1, 4,1, 2,2] + # - [16, 16, 16, 1, 1, 2,8, 2,2] + # - [16, 16, 16, 1, 1, 2,4, 2,2] + # - [16, 16, 16, 1, 1, 2,2, 2,2] + # - [16, 16, 16, 1, 1, 2,1, 2,2] + # - [16, 16, 16, 1, 1, 1,8, 2,2] + # - [16, 16, 16, 1, 1, 1,4, 2,2] + # - [16, 16, 16, 1, 1, 1,2, 2,2] + # - [16, 16, 16, 1, 1, 1,1, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [4] + # # - VectorWidthB: [4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + + - # HGEMM TN - Test DepthU, WGM, VW + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 32, 64 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [2, 4, 8] + - GlobalReadVectorWidthB: [2, 4, 8] + - GlobalSplitU: [0] + # - LocalReadVectorWidth: [8] + - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + - [16, 16, 16, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + - StreamK: [3] + - TransposeLDS: [0] + # - VectorWidthA: [1, 2, 4] + # - VectorWidthB: [1, 2, 4] + - WorkGroupMapping: [0, 1, 2, 4, 8, 16, 32, 64] # works + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + - Exact: [2055, 2055, 1, 1031] + + # - # HGEMM TN - Test tuning params + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [0, 1] + # - DepthU: [64] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [8] + # - GlobalReadVectorWidthB: [8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - StaggerU: [0, 32] + # - ScheduleIterAlg: [3] + # - SourceSwap: [False, True] + # - StoreRemapVectorWidth: [0, 4] + # # - StoreVectorWidth: [1] + # - StreamK: [3] + # - PrefetchLocalRead: [1, 3] + # - NonTemporalC: [0, 7] + # - NonTemporalD: [0, 7] + # - TransposeLDS: [0, 1] + # # - VectorWidthA: [1] + # # - VectorWidthB: [1] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # # - Exact: [4103, 4103, 1, 1031] + # - Exact: [2055, 2055, 1, 1031] + + # - # HGEMM TN - Test size range + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [2, 4, 8] + # - GlobalReadVectorWidthB: [2, 4, 8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1, 2, 4] + # # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [1, 2, 4] + # # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Range: [ [32, 249, 2048], [32, 249, 2048], [1], [32, 249, 2048] ] + + - # HGEMM TT + - # ProblemType + OperationType: GEMM + DataType: h + DestDataType: h + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: True + UseBeta: True + Batched: True + + # - # HGEMM TT - Test MatrixInstruction variants + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [ 64 ] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [8] + # - GlobalReadVectorWidthB: [8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [8] + # - MatrixInstruction: + # - [32, 32, 8, 1, 1, 4,4, 2,2] + # - [32, 32, 8, 1, 1, 3,3, 2,2] + # - [32, 32, 8, 1, 1, 2,2, 2,2] + # - [32, 32, 8, 1, 1, 1,1, 2,2] + # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 8,4, 2,2] + # - [16, 16, 16, 1, 1, 8,2, 2,2] + # - [16, 16, 16, 1, 1, 8,1, 2,2] + # - [16, 16, 16, 1, 1, 4,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - [16, 16, 16, 1, 1, 4,2, 2,2] + # - [16, 16, 16, 1, 1, 4,1, 2,2] + # - [16, 16, 16, 1, 1, 2,8, 2,2] + # - [16, 16, 16, 1, 1, 2,4, 2,2] + # - [16, 16, 16, 1, 1, 2,2, 2,2] + # - [16, 16, 16, 1, 1, 2,1, 2,2] + # - [16, 16, 16, 1, 1, 1,8, 2,2] + # - [16, 16, 16, 1, 1, 1,4, 2,2] + # - [16, 16, 16, 1, 1, 1,2, 2,2] + # - [16, 16, 16, 1, 1, 1,1, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [4] + # # - VectorWidthB: [4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + + # - # HGEMM TT - Test DepthU, WGM, VW + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [ 32, 64 ] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [2, 4, 8] + # - GlobalReadVectorWidthB: [2, 4, 8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [8] + # - MatrixInstruction: + # # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [1, 2, 4] + # # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [0, 1, 2, 4, 8, 16, 32, 64] # works + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # # - Exact: [4103, 4103, 1, 1031] + # - Exact: [2055, 2055, 1, 1031] + + - # HGEMM TT - Test tuning params + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [0, 1] + - DepthU: [64] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [8] + - GlobalReadVectorWidthB: [8] + - GlobalSplitU: [0] + # - LocalReadVectorWidth: [1] + - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + - [16, 16, 16, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - StaggerU: [0, 32] + - ScheduleIterAlg: [3] + - SourceSwap: [False, True] + - StoreRemapVectorWidth: [0, 4] + # - StoreVectorWidth: [1] + - StreamK: [3] + - PrefetchLocalRead: [1, 3] + - NonTemporalC: [0, 7] + - NonTemporalD: [0, 7] + - TransposeLDS: [0, 1] + # - VectorWidthA: [1] + # - VectorWidthB: [1] + - WorkGroupMapping: [1] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + - Exact: [2055, 2055, 1, 1031] + + # - # HGEMM TT - Test size range + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [2, 4, 8] + # - GlobalReadVectorWidthB: [2, 4, 8] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1, 2, 4] + # # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # - [16, 16, 16, 1, 1, 8,8, 2,2] + # - [16, 16, 16, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # # - VectorWidthA: [1, 2, 4] + # # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Range: [ [32, 249, 2048], [32, 249, 2048], [1], [32, 249, 2048] ] diff --git a/tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml b/tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml new file mode 100644 index 0000000000..f4e9b61794 --- /dev/null +++ b/tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml @@ -0,0 +1,793 @@ + +GlobalParameters: + NumElementsToValidate: -1 + BoundsCheck: False + KernelTime: False + DataInitTypeAlpha: 1 + DataInitTypeBeta: 1 + DataInitTypeA: 12 + DataInitTypeB: 13 + DataInitTypeC: 12 + # DataInitTypeC: 1 + # ValidationPrintValids: True + MaxWorkspaceSize: 134217728 + # PrintSolutionRejectionReason: True + # ForceGenerateKernel: True + # GenerateSourcesAndExit: True + NumWarmups: 0 + EnqueuesPerSync: 1 + # NumBenchmarks: 10 + +BenchmarkProblems: + + - # SGEMM NT + - # ProblemType + OperationType: GEMM + DataType: s + TransposeA: False + TransposeB: True + UseBeta: True + Batched: True + + - # SGEMM NT - Test MatrixInstruction variants + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 16 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [1] + - GlobalReadVectorWidthB: [1] + - GlobalSplitU: [0] + - LocalReadVectorWidth: [1] + - MatrixInstruction: + - [32, 32, 2, 1, 1, 4,4, 2,2] + - [32, 32, 2, 1, 1, 3,3, 2,2] + - [32, 32, 2, 1, 1, 2,2, 2,2] + - [32, 32, 2, 1, 1, 1,1, 2,2] + - [16, 16, 4, 1, 1, 8,8, 2,2] + - [16, 16, 4, 1, 1, 8,4, 2,2] + - [16, 16, 4, 1, 1, 8,2, 2,2] + - [16, 16, 4, 1, 1, 8,1, 2,2] + - [16, 16, 4, 1, 1, 4,8, 2,2] + - [16, 16, 4, 1, 1, 4,4, 2,2] + - [16, 16, 4, 1, 1, 4,2, 2,2] + - [16, 16, 4, 1, 1, 4,1, 2,2] + - [16, 16, 4, 1, 1, 2,8, 2,2] + - [16, 16, 4, 1, 1, 2,4, 2,2] + - [16, 16, 4, 1, 1, 2,2, 2,2] + - [16, 16, 4, 1, 1, 2,1, 2,2] + - [16, 16, 4, 1, 1, 1,8, 2,2] + - [16, 16, 4, 1, 1, 1,4, 2,2] + - [16, 16, 4, 1, 1, 1,2, 2,2] + - [16, 16, 4, 1, 1, 1,1, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + - StreamK: [3] + - TransposeLDS: [0] + - VectorWidthA: [1] + - VectorWidthB: [1] + - WorkGroupMapping: [1] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + - Exact: [4103, 4103, 1, 1031] + + - # SGEMM NT - Test DepthU, WGM, VW + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 8, 16, 32 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [1, 2, 4] + - GlobalReadVectorWidthB: [1, 2, 4] + - GlobalSplitU: [0] + - LocalReadVectorWidth: [1] + - MatrixInstruction: + # - [16, 16, 4, 1, 1, 8,8, 2,2] + - [16, 16, 4, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + - StoreVectorWidth: [4] + - StreamK: [3] + - TransposeLDS: [0] + - VectorWidthA: [1, 2, 4] + - VectorWidthB: [1, 2, 4] + - WorkGroupMapping: [0, 1, 2, 4, 8, 16, 32, 64] # works + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + - Exact: [2055, 2055, 1, 1031] + + - # SGEMM NT - Test tuning params + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [0, 1] + - DepthU: [16] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [4] + - GlobalReadVectorWidthB: [4] + - GlobalSplitU: [0] + - LocalReadVectorWidth: [1] + - MatrixInstruction: + # - [16, 16, 4, 1, 1, 8,8, 2,2] + - [16, 16, 4, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - StaggerU: [0, 32] + - ScheduleIterAlg: [3] + - SourceSwap: [False, True] + - StoreRemapVectorWidth: [0, 4] + - StoreVectorWidth: [1] + - StreamK: [3] + - PrefetchLocalRead: [1, 3] + - NonTemporalC: [0, 7] + - NonTemporalD: [0, 7] + - TransposeLDS: [0] + - VectorWidthA: [4] + - VectorWidthB: [4] + - WorkGroupMapping: [1] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + - Exact: [2055, 2055, 1, 1031] + + # - # SGEMM NT - Test size range + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [4] + # - GlobalReadVectorWidthB: [4] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1, 2, 4] + # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # # - [16, 16, 4, 1, 1, 8,8, 2,2] + # - [16, 16, 4, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # - VectorWidthA: [4] + # - VectorWidthB: [4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Range: [ [32, 249, 2048], [32, 249, 2048], [1], [32, 249, 2048] ] + + - # SGEMM NN + - # ProblemType + OperationType: GEMM + DataType: s + TransposeA: False + TransposeB: False + UseBeta: True + Batched: True + + - # SGEMM NN - Test MatrixInstruction variants + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 16 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [1] + - GlobalReadVectorWidthB: [1] + - GlobalSplitU: [0] + - LocalReadVectorWidth: [1] + - MatrixInstruction: + - [32, 32, 2, 1, 1, 4,4, 2,2] + - [32, 32, 2, 1, 1, 3,3, 2,2] + - [32, 32, 2, 1, 1, 2,2, 2,2] + - [32, 32, 2, 1, 1, 1,1, 2,2] + - [16, 16, 4, 1, 1, 8,8, 2,2] + - [16, 16, 4, 1, 1, 8,4, 2,2] + - [16, 16, 4, 1, 1, 8,2, 2,2] + - [16, 16, 4, 1, 1, 8,1, 2,2] + - [16, 16, 4, 1, 1, 4,8, 2,2] + - [16, 16, 4, 1, 1, 4,4, 2,2] + - [16, 16, 4, 1, 1, 4,2, 2,2] + - [16, 16, 4, 1, 1, 4,1, 2,2] + - [16, 16, 4, 1, 1, 2,8, 2,2] + - [16, 16, 4, 1, 1, 2,4, 2,2] + - [16, 16, 4, 1, 1, 2,2, 2,2] + - [16, 16, 4, 1, 1, 2,1, 2,2] + - [16, 16, 4, 1, 1, 1,8, 2,2] + - [16, 16, 4, 1, 1, 1,4, 2,2] + - [16, 16, 4, 1, 1, 1,2, 2,2] + - [16, 16, 4, 1, 1, 1,1, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + - StreamK: [3] + - TransposeLDS: [0] + - VectorWidthA: [1] + - VectorWidthB: [1] + - WorkGroupMapping: [1] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + - Exact: [4103, 4103, 1, 1031] + + # - # SGEMM NN - Test DepthU, WGM, VW + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [ 8, 16, 32 ] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [1, 2, 4] + # - GlobalReadVectorWidthB: [1, 2, 4] + # - GlobalSplitU: [0] + # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # # - [16, 16, 4, 1, 1, 8,8, 2,2] + # - [16, 16, 4, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # - VectorWidthA: [1, 2, 4] + # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [0, 1, 2, 4, 8, 16, 32, 64] # works + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # # - Exact: [4103, 4103, 1, 1031] + # - Exact: [2055, 2055, 1, 1031] + + # - # SGEMM NN - Test tuning params + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [0, 1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [4] + # - GlobalReadVectorWidthB: [4] + # - GlobalSplitU: [0] + # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # # - [16, 16, 4, 1, 1, 8,8, 2,2] + # - [16, 16, 4, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - StaggerU: [0, 32] + # - ScheduleIterAlg: [3] + # - SourceSwap: [False, True] + # - StoreRemapVectorWidth: [0, 4] + # - StoreVectorWidth: [1] + # - StreamK: [3] + # - PrefetchLocalRead: [1, 3] + # - NonTemporalC: [0, 7] + # - NonTemporalD: [0, 7] + # - TransposeLDS: [0, 1] + # - VectorWidthA: [4] + # - VectorWidthB: [4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # # - Exact: [4103, 4103, 1, 1031] + # - Exact: [2055, 2055, 1, 1031] + + # - # SGEMM NN - Test size range + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [4] + # - GlobalReadVectorWidthB: [4] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1, 2, 4] + # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # # - [16, 16, 4, 1, 1, 8,8, 2,2] + # - [16, 16, 4, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # - VectorWidthA: [4] + # - VectorWidthB: [4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Range: [ [32, 249, 2048], [32, 249, 2048], [1], [32, 249, 2048] ] + + - # SGEMM TN + - # ProblemType + OperationType: GEMM + DataType: s + TransposeA: True + TransposeB: False + UseBeta: True + Batched: True + + # - # SGEMM TN - Test MatrixInstruction variants + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [ 16 ] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [1] + # - GlobalReadVectorWidthB: [1] + # - GlobalSplitU: [0] + # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # - [32, 32, 2, 1, 1, 4,4, 2,2] + # - [32, 32, 2, 1, 1, 3,3, 2,2] + # - [32, 32, 2, 1, 1, 2,2, 2,2] + # - [32, 32, 2, 1, 1, 1,1, 2,2] + # - [16, 16, 4, 1, 1, 8,8, 2,2] + # - [16, 16, 4, 1, 1, 8,4, 2,2] + # - [16, 16, 4, 1, 1, 8,2, 2,2] + # - [16, 16, 4, 1, 1, 8,1, 2,2] + # - [16, 16, 4, 1, 1, 4,8, 2,2] + # - [16, 16, 4, 1, 1, 4,4, 2,2] + # - [16, 16, 4, 1, 1, 4,2, 2,2] + # - [16, 16, 4, 1, 1, 4,1, 2,2] + # - [16, 16, 4, 1, 1, 2,8, 2,2] + # - [16, 16, 4, 1, 1, 2,4, 2,2] + # - [16, 16, 4, 1, 1, 2,2, 2,2] + # - [16, 16, 4, 1, 1, 2,1, 2,2] + # - [16, 16, 4, 1, 1, 1,8, 2,2] + # - [16, 16, 4, 1, 1, 1,4, 2,2] + # - [16, 16, 4, 1, 1, 1,2, 2,2] + # - [16, 16, 4, 1, 1, 1,1, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # - VectorWidthA: [1] + # - VectorWidthB: [1] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + + - # SGEMM TN - Test DepthU, WGM, VW + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [ 8, 16, 32 ] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [1, 2, 4] + - GlobalReadVectorWidthB: [1, 2, 4] + - GlobalSplitU: [0] + - LocalReadVectorWidth: [1] + - MatrixInstruction: + # - [16, 16, 4, 1, 1, 8,8, 2,2] + - [16, 16, 4, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - ScheduleIterAlg: [3] + - SourceSwap: [True] + - StoreRemapVectorWidth: [0] + - StoreVectorWidth: [4] + - StreamK: [3] + - TransposeLDS: [0] + - VectorWidthA: [1, 2, 4] + - VectorWidthB: [1, 2, 4] + - WorkGroupMapping: [0, 1, 2, 4, 8, 16, 32, 64] # works + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + - Exact: [2055, 2055, 1, 1031] + + # - # SGEMM TN - Test tuning params + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [0, 1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [4] + # - GlobalReadVectorWidthB: [4] + # - GlobalSplitU: [0] + # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # # - [16, 16, 4, 1, 1, 8,8, 2,2] + # - [16, 16, 4, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - StaggerU: [0, 32] + # - ScheduleIterAlg: [3] + # - SourceSwap: [False, True] + # - StoreRemapVectorWidth: [0, 4] + # - StoreVectorWidth: [1] + # - StreamK: [3] + # - PrefetchLocalRead: [1, 3] + # - NonTemporalC: [0, 7] + # - NonTemporalD: [0, 7] + # - TransposeLDS: [0, 1] + # - VectorWidthA: [4] + # - VectorWidthB: [4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # # - Exact: [4103, 4103, 1, 1031] + # - Exact: [2055, 2055, 1, 1031] + + # - # SGEMM TN - Test size range + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [4] + # - GlobalReadVectorWidthB: [4] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1, 2, 4] + # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # # - [16, 16, 4, 1, 1, 8,8, 2,2] + # - [16, 16, 4, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # - VectorWidthA: [4] + # - VectorWidthB: [4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Range: [ [32, 249, 2048], [32, 249, 2048], [1], [32, 249, 2048] ] + + - # SGEMM TT + - # ProblemType + OperationType: GEMM + DataType: s + TransposeA: True + TransposeB: True + UseBeta: True + Batched: True + + # - # SGEMM TT - Test MatrixInstruction variants + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [ 16 ] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [1] + # - GlobalReadVectorWidthB: [1] + # - GlobalSplitU: [0] + # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # - [32, 32, 2, 1, 1, 4,4, 2,2] + # - [32, 32, 2, 1, 1, 3,3, 2,2] + # - [32, 32, 2, 1, 1, 2,2, 2,2] + # - [32, 32, 2, 1, 1, 1,1, 2,2] + # - [16, 16, 4, 1, 1, 8,8, 2,2] + # - [16, 16, 4, 1, 1, 8,4, 2,2] + # - [16, 16, 4, 1, 1, 8,2, 2,2] + # - [16, 16, 4, 1, 1, 8,1, 2,2] + # - [16, 16, 4, 1, 1, 4,8, 2,2] + # - [16, 16, 4, 1, 1, 4,4, 2,2] + # - [16, 16, 4, 1, 1, 4,2, 2,2] + # - [16, 16, 4, 1, 1, 4,1, 2,2] + # - [16, 16, 4, 1, 1, 2,8, 2,2] + # - [16, 16, 4, 1, 1, 2,4, 2,2] + # - [16, 16, 4, 1, 1, 2,2, 2,2] + # - [16, 16, 4, 1, 1, 2,1, 2,2] + # - [16, 16, 4, 1, 1, 1,8, 2,2] + # - [16, 16, 4, 1, 1, 1,4, 2,2] + # - [16, 16, 4, 1, 1, 1,2, 2,2] + # - [16, 16, 4, 1, 1, 1,1, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # - VectorWidthA: [1] + # - VectorWidthB: [1] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + + # - # SGEMM TT - Test DepthU, WGM, VW + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [ 8, 16, 32 ] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [1, 2, 4] + # - GlobalReadVectorWidthB: [1, 2, 4] + # - GlobalSplitU: [0] + # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # # - [16, 16, 4, 1, 1, 8,8, 2,2] + # - [16, 16, 4, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # - VectorWidthA: [1, 2, 4] + # - VectorWidthB: [1, 2, 4] + # - WorkGroupMapping: [0, 1, 2, 4, 8, 16, 32, 64] # works + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Exact: [512, 512, 1, 512] + # # - Exact: [4096, 4096, 1, 1024] + # # - Exact: [4103, 4096, 1, 1024] + # # - Exact: [4096, 4103, 1, 1024] + # # - Exact: [4096, 4096, 1, 1031] + # # - Exact: [4103, 4103, 1, 1031] + # - Exact: [2055, 2055, 1, 1031] + + - # SGEMM TT - Test tuning params + InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: ["Assembly"] + - PrefetchLocalRead: [True] + ForkParameters: + - 1LDSBuffer: [0, 1] + - DepthU: [16] + - ExpandPointerSwap: [False] + - GlobalReadVectorWidthA: [4] + - GlobalReadVectorWidthB: [4] + - GlobalSplitU: [0] + - LocalReadVectorWidth: [1] + - MatrixInstruction: + # - [16, 16, 4, 1, 1, 8,8, 2,2] + - [16, 16, 4, 1, 1, 4,4, 2,2] + - MIArchVgpr: [0] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - StaggerU: [0, 32] + - ScheduleIterAlg: [3] + - SourceSwap: [False, True] + - StoreRemapVectorWidth: [0, 4] + - StoreVectorWidth: [1] + - StreamK: [3] + - PrefetchLocalRead: [1, 3] + - NonTemporalC: [0, 7] + - NonTemporalD: [0, 7] + - TransposeLDS: [0, 1] + - VectorWidthA: [4] + - VectorWidthB: [4] + - WorkGroupMapping: [1] + + BenchmarkForkParameters: + JoinParameters: + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Exact: [512, 512, 1, 512] + # - Exact: [4096, 4096, 1, 1024] + # - Exact: [4103, 4096, 1, 1024] + # - Exact: [4096, 4103, 1, 1024] + # - Exact: [4096, 4096, 1, 1031] + # - Exact: [4103, 4103, 1, 1031] + - Exact: [2055, 2055, 1, 1031] + + # - # SGEMM TT - Test size range + # InitialSolutionParameters: + # BenchmarkCommonParameters: + # - KernelLanguage: ["Assembly"] + # - PrefetchLocalRead: [True] + # ForkParameters: + # - 1LDSBuffer: [1] + # - DepthU: [16] + # - ExpandPointerSwap: [False] + # - GlobalReadVectorWidthA: [4] + # - GlobalReadVectorWidthB: [4] + # - GlobalSplitU: [0] + # # - LocalReadVectorWidth: [1, 2, 4] + # - LocalReadVectorWidth: [1] + # - MatrixInstruction: + # # - [16, 16, 4, 1, 1, 8,8, 2,2] + # - [16, 16, 4, 1, 1, 4,4, 2,2] + # - MIArchVgpr: [0] + # - PrefetchGlobalRead: [2] + # - PrefetchLocalRead: [1] + # - ScheduleIterAlg: [3] + # - SourceSwap: [True] + # - StoreRemapVectorWidth: [0] + # - StoreVectorWidth: [4] + # - StreamK: [3] + # - TransposeLDS: [0] + # - VectorWidthA: [4] + # - VectorWidthB: [4] + # - WorkGroupMapping: [1] + + # BenchmarkForkParameters: + # JoinParameters: + # BenchmarkJoinParameters: + # BenchmarkFinalParameters: + # - ProblemSizes: + # - Range: [ [32, 249, 2048], [32, 249, 2048], [1], [32, 249, 2048] ] From 7b84b912a7d5ada373e95e90194f1f465f0df8b4 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Thu, 22 Aug 2024 14:55:29 +0000 Subject: [PATCH 27/36] Bug fixes from merge --- tensilelite/Tensile/Components/GSU.py | 15 ++++++++------- tensilelite/Tensile/KernelWriter.py | 2 -- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tensilelite/Tensile/Components/GSU.py b/tensilelite/Tensile/Components/GSU.py index 770546c910..fdaaadad57 100644 --- a/tensilelite/Tensile/Components/GSU.py +++ b/tensilelite/Tensile/Components/GSU.py @@ -163,7 +163,7 @@ def graWorkGroup(self, writer, kernel): module.add(SCBranchSCC1(labelName=gsuLabel.getLabelName(), comment="branch if GSU == 1")) if ((kernel["GlobalSplitUAlgorithm"] == 'MultipleBufferSingleKernel')): - extReadEpilogueLabeltmp = Label(label=self.labels.getNameInc("LoadExternalEpilogueStruct"), comment="") + extReadEpilogueLabeltmp = Label(label=writer.labels.getNameInc("LoadExternalEpilogueStruct"), comment="") module.addComment0("Check if custom structure pointer is null") if kernel["ProblemType"]["SupportUserArgs"]: module.add(SCmpEQU32(src0=sgpr("ArgType"), src1=2, comment="ArgType == 2 ?")) @@ -198,6 +198,7 @@ def graWorkGroup(self, writer, kernel): def computeLoadSrd(self, writer, kernel, tP, stmp, tileStart): module = Module("GSU On computeLoadSrd") + tc = tP["tensorChar"] depthU = kernel["DepthU"] depthUDiv = kernel["DepthU"] gsuOffsetStr = "gsuOffset = DepthU*bpeGR*GSUSumIdx" @@ -213,20 +214,20 @@ def computeLoadSrd(self, writer, kernel, tP, stmp, tileStart): gsuOffsetStr = "gsuOffset = DepthU/%s*bpeGR*GSUSumIdx"%(divider) if writer.states.gsu_wg_coalesced: gsuOffsetStr = "gsuOffset = DepthU*accumulatedNumOfLoopCounterL" - loopCounterName = writer.loopCounterName(kernel, self.states.unrollIdx) + loopCounterName = writer.loopCounterName(kernel, writer.states.unrollIdx) module.add(SLShiftRightB32(dst=sgpr(loopCounterName), src=sgpr("SizesSum"), shiftHex=log2(depthU), \ comment="s[%s] = s[sgprSizesSum] / %s"%(loopCounterName, depthU))) module.add(writer.calculateLoopNumIterOffsetGsu(kernel, loopCounterName, tmpSgprInfo)) - module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(stmp+0), sgpr(stmp+1), sgpr(stmp+0), depthUDiv, gsuOffsetStr)) + module.addModuleAsFlatItems(writer.s_mul_u64_u32(sgpr(stmp+0), sgpr(stmp+1), sgpr(stmp+0), depthUDiv, gsuOffsetStr)) else: gsuOffsetStr = "gsuOffset = DepthU*GSUSumIdx" - module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(stmp+0), sgpr(stmp+1), depthUDiv, sgpr("GSUSumIdx"), gsuOffsetStr)) + module.addModuleAsFlatItems(writer.s_mul_u64_u32(sgpr(stmp+0), sgpr(stmp+1), depthUDiv, sgpr("GSUSumIdx"), gsuOffsetStr)) unrollSummation = [ i for i in tP["ia"] if i in kernel["ProblemType"]["IndicesSummation"] ] stride = writer.strideRef(tc, unrollSummation[-1]) - if tP["tlu"] and not self.isConstUnitStride(stride): + if tP["tlu"] and not writer.isConstUnitStride(stride): # non-transpose case, unroll is in perp dim and should be scaled by unroll Stride - module.addModuleAsFlatItems(self.s_mul_u64_u32(sgpr(stmp), sgpr(stmp+1), sgpr(stmp+0), \ + module.addModuleAsFlatItems(writer.s_mul_u64_u32(sgpr(stmp), sgpr(stmp+1), sgpr(stmp+0), \ stride, "tlu=1, scaled unroll-offset by stride")) module.add(SAddU32(dst=sgpr(tileStart+0), src0=sgpr(tileStart+0), src1=sgpr(stmp+0), comment="accum GsuOffset term to tilestart")) @@ -411,7 +412,7 @@ def tailLoopNumIter(self, writer, kernel, loopCounter): if writer.states.gsu_wg_coalesced: depthU = kernel["DepthU"] # calculate the lastWg - with self.allocTmpSgpr(2) as tmpSgprInfo: + with writer.allocTmpSgpr(2) as tmpSgprInfo: tmpSgpr = tmpSgprInfo.idx tmpVgpr = writer.vgprPool.checkOut(2,"tmp") tmpVgprRes = RegisterPoolResource(idx=tmpVgpr, size=2) diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index b2a8fde1fe..c0fceecbbd 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -3763,8 +3763,6 @@ def readWriteVectors(mat, vw, kernel): self.states.nonPostLoopSgpr.remove("AddressB") self.states.nonPostLoopSgpr.remove("StridesA") self.states.nonPostLoopSgpr.remove("StridesB") - self.states.nonPostLoopSgpr.remove("NumWorkGroups0") - self.states.nonPostLoopSgpr.remove("NumWorkGroups1") self.states.preloadScaleA = False self.states.preloadScaleB = False From cfdaf49bba39a00af0879cf5f4cb2a62be5cc73c Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Fri, 23 Aug 2024 17:16:16 +0000 Subject: [PATCH 28/36] Update targets for new tests --- tensilelite/Tensile/Tests/common/gsu/f32_gsu.yaml | 2 ++ tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml | 2 ++ tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml | 2 ++ tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/tensilelite/Tensile/Tests/common/gsu/f32_gsu.yaml b/tensilelite/Tensile/Tests/common/gsu/f32_gsu.yaml index 7c296f842d..70135974c7 100644 --- a/tensilelite/Tensile/Tests/common/gsu/f32_gsu.yaml +++ b/tensilelite/Tensile/Tests/common/gsu/f32_gsu.yaml @@ -1,3 +1,5 @@ +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch GlobalParameters: NumElementsToValidate: -1 diff --git a/tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml b/tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml index daf5843091..5e8b90170c 100644 --- a/tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml +++ b/tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml @@ -1,3 +1,5 @@ +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch GlobalParameters: NumElementsToValidate: -1 diff --git a/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml b/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml index 72525a7919..e83e49cf97 100644 --- a/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml +++ b/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml @@ -1,3 +1,5 @@ +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch GlobalParameters: NumElementsToValidate: -1 diff --git a/tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml b/tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml index f4e9b61794..2e991675bc 100644 --- a/tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml +++ b/tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml @@ -1,3 +1,5 @@ +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch GlobalParameters: NumElementsToValidate: -1 From 44f2d357f037d89fb00f77f9bd7f36a6f1454601 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Fri, 23 Aug 2024 17:20:41 +0000 Subject: [PATCH 29/36] Remove duplicate code and addreject for unsupported config --- tensilelite/Tensile/SolutionStructs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index f24bb94964..c1c1059081 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -1940,7 +1940,6 @@ def assignDerivedParameters(state): computeName = state["ProblemType"]["ComputeDataType"].toName() if state["StreamK"] > 0 and state["StreamKAtomic"] == 0: # StreamK Workspace size - computeBytes = state["ProblemType"]["ComputeDataType"].numBytes() state["_GlobalAccumulation"] = 'PartialsBuffer' elif state["GlobalSplitUAlgorithm"] == 'SingleBuffer': if computeName != state["ProblemType"]["DestDataType"].toName(): @@ -1963,7 +1962,9 @@ def assignDerivedParameters(state): if state["StreamK"] != 0: if state["MIWaveGroup"][0] * state["MIWaveGroup"][1] != 4: reject(state, "Stream-K requries MIWaveGroup0*MIWaveGroup1=4") - if state["EnableMatrixInstruction"] and globalParameters["AsmCaps"][isa]["HasWMMA"]: + if not state["EnableMatrixInstruction"]: + reject(state, "Stream-K requires MatrixInstruction") + if globalParameters["AsmCaps"][isa]["HasWMMA"]: reject(state, "Stream-K untested with WMMA") if state["GlobalSplitU"] > 0: reject(state, "Cannot enable both Stream-K and GSU") From e7d7d571d1459a090f662f629bc7307303d381e2 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Fri, 23 Aug 2024 17:32:15 +0000 Subject: [PATCH 30/36] If stream-k enabled, force GSU=0 --- tensilelite/Tensile/SolutionStructs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs.py index c1c1059081..cdb0a30a5e 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs.py @@ -1960,14 +1960,13 @@ def assignDerivedParameters(state): isa = tuple(state["ISA"]) if state["StreamK"] != 0: + state["GlobalSplitU"] = 0 # Cannot enable both Stream-K and GSU if state["MIWaveGroup"][0] * state["MIWaveGroup"][1] != 4: reject(state, "Stream-K requries MIWaveGroup0*MIWaveGroup1=4") if not state["EnableMatrixInstruction"]: reject(state, "Stream-K requires MatrixInstruction") if globalParameters["AsmCaps"][isa]["HasWMMA"]: reject(state, "Stream-K untested with WMMA") - if state["GlobalSplitU"] > 0: - reject(state, "Cannot enable both Stream-K and GSU") # if state["PersistentKernel"]: # reject(state, "Cannot enable both Stream-K and PersistentKernel") if not state["ProblemType"]["StridedBatched"]: From bfbfde1e1417406f9cccc86cee80a70703601cd8 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Wed, 28 Aug 2024 17:39:06 +0000 Subject: [PATCH 31/36] Bug fixes for SKXCCM and SK=2 --- tensilelite/Tensile/Components/StreamK.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index 184d4978b9..f721fdd8b7 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -61,12 +61,11 @@ def __call__(self, writer, kernel): sGridM = tmpSgprRes.idx + 3 sTmp = None sTmpRes = None - sqTmp = None + sqTmp = writer.sgprPool.checkOut(1, "sqTmp", preventOverflow=False) divisor = kernel["StreamKXCCMapping"] if ((divisor & (divisor - 1)) != 0): # Need temp registers if not power of 2 sTmp = writer.sgprPool.checkOut(2, "sTmp", preventOverflow=False) sTmpRes = RegisterPoolResource(idx=sTmp, size=2) - sqTmp = writer.sgprPool.checkOut(1, "sqTmp", preventOverflow=False) # sGridC = ceil(grid / xccm) module.add(SAddU32(dst=sgpr(sXCC), src0=sgpr("skGrid"), src1=hex(kernel["StreamKXCCMapping"] - 1), comment="ceil(grid/xccm)")) @@ -87,9 +86,10 @@ def __call__(self, writer, kernel): module.add(SMulI32(dst=sgpr(sXCC), src0=sgpr(sXCC), src1=sgpr(sGridC), comment="XCC group id")) module.add(SAddU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=sgpr(sXCC), comment="Add XCC group offset")) module.add(SAddU32(dst=sgpr("WorkGroup0"), src0=sgpr("WorkGroup0"), src1=sgpr(sGridM), comment="Add remainder offset")) + + writer.sgprPool.checkIn(sqTmp) if sTmp is not None: writer.sgprPool.checkIn(sTmp) - writer.sgprPool.checkIn(sqTmp) return module @@ -1789,7 +1789,7 @@ def calculateLoopNumIter(self, writer, kernel, loopCounterName, loopIdx, tmpSgpr def storeBranches(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct): module = Module("StreamK TwoTileOriginal storeBranches") - module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, element, tmpVgpr, cvtVgprStruct)) + module.add(self.storeBranchesCommon(writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct)) return module def writePartials(self, writer, kernel, skPartialsLabel, vectorWidths, elements, tmpVgpr, cvtVgprStruct, endLabel): From e6de95fb7e18821af2d6a2aee0e22d698845890a Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Fri, 6 Sep 2024 15:23:11 +0000 Subject: [PATCH 32/36] Fix race condition when resetting synchronizer flags --- tensilelite/Tensile/Components/StreamK.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index f721fdd8b7..7460420a5b 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -24,7 +24,7 @@ SCmpLtU32, SCSelectB32, sMagicDivAlg2, SMulI32, SSubU32, SMinU32, SMovB32, SCBranchSCC1, SCmpLeU32, VMovB32, vgpr, \ SAddCU32, SCmpGtU32, SCMovB32, SAddI32, SCmpEQU32, SCBranchSCC0, SLShiftLeftB32, SLoadB32, SWaitCnt, SMEMModifiers, \ log2, SBarrier, SStoreB32, SLongBranchPositive, SBranch, ceilDivide, replaceHolder, SNop, staticMultiply, SSleep, \ - VAddF32, VAddF64, SAndB32, SLShiftRightB32 + VAddF32, VAddF64, SAndB32, SLShiftRightB32, VReadfirstlaneB32 from ..Common import print2 # from ..TensileInstructions.Containers import SMEMModifiers from ..Component import Component @@ -300,8 +300,13 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele # TODO Barrier here to sync all threads in workgroup, but maybe better to have separate flag for each wavefront (to be tested) module.add(SBarrier(comment="wait for all workgroups before resetting flag")) + skipFlagReset = Label(label=writer.labels.getNameInc("SK_SkipFlagReset"), comment="") + module.add(VReadfirstlaneB32(dst=sgpr(tmpSgpr+2), src=vgpr("Serial"), comment="Wave 0 updates flags")) + module.add(SCmpEQU32(src0=sgpr(tmpSgpr+2), src1=0, comment="Check for wave 0")) + module.add(SCBranchSCC0(labelName=skipFlagReset.getLabelName(), comment="Skip flag reset")) module.add(SMovB32(dst=sgpr(tmpSgpr+2), src=0, comment="flag data")) module.add(SStoreB32(src=sgpr(tmpSgpr+2), base=sgpr("AddressFlags", 2), soffset=sgpr(tmpSgpr), smem=SMEMModifiers(glc=1), comment="reset flag")) + module.add(skipFlagReset) writer.sgprPool.checkIn(tmpSgpr) fixupEdge = [False] # Test no edge variant @@ -596,8 +601,13 @@ def partialsWriteProcedure(self, writer, kernel, vectorWidths, elements, alpha, module.add(SLShiftLeftB32(dst=sgpr(tmpSgpr), src=sgpr("StreamKIdx"), shiftHex=log2(4), comment="flag offset based on CTA index")) with writer.allocTmpSgpr(1) as flagSgprRes: flagSgpr = flagSgprRes.idx + skipFlagSet = Label(label=writer.labels.getNameInc("SK_SkipFlagSet"), comment="") + module.add(VReadfirstlaneB32(dst=sgpr(flagSgpr), src=vgpr("Serial"), comment="Wave 0 updates flags")) + module.add(SCmpEQU32(src0=sgpr(flagSgpr), src1=0, comment="Check for wave 0")) + module.add(SCBranchSCC0(labelName=skipFlagSet.getLabelName(), comment="Skip flag set")) module.add(SMovB32(dst=sgpr(flagSgpr), src=1, comment="flag data")) module.add(SStoreB32(src=sgpr(flagSgpr), base=sgpr("AddressFlags", 2), soffset=sgpr(tmpSgpr), smem=SMEMModifiers(glc=1), comment="set flag")) + module.add(skipFlagSet) module.add(SWaitCnt(lgkmcnt=0, comment="wait for flag")) # TODO just for testing # TODO - if this is the last tile, don't need to jump to next instruction From fc61d879ddc10fddbf332bb0a89afe93b31baf20 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Fri, 6 Sep 2024 16:04:54 +0000 Subject: [PATCH 33/36] Disable LRA backup, not needed for stream-k, and not compatible with current vgpr allocations --- .../Tensile/Components/PersistentLoop.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/tensilelite/Tensile/Components/PersistentLoop.py b/tensilelite/Tensile/Components/PersistentLoop.py index bfd42bab44..07ca15f8d4 100644 --- a/tensilelite/Tensile/Components/PersistentLoop.py +++ b/tensilelite/Tensile/Components/PersistentLoop.py @@ -103,16 +103,22 @@ def recalcLocalReadAddressesAB(self, writer, kernel): needRecalc = writer.states.numReadsIterCoalescedA > 1 or writer.states.numReadsIterCoalescedB > 1 # backup LocalReadAddr # LdsPad + LBSPP case, need to backup LocalReadAddr even if recalc is not done - needBackupLRAddr = needRecalc or (kernel["LdsPadA"] and kernel["LdsBlockSizePerPadA"] or kernel["LdsPadB"] and kernel["LdsBlockSizePerPadB"]) - - if needBackupLRAddr: - # need to back-up the LRA before reCalculation for wider local read (when no wlr, no need to do this) - if writer.oriLraA is None: # and not kernel["DirectToVgprA"]: # no local read code if DirectToVgpr is enabled - writer.oriLraA = writer.vgprPool.checkOut(1, "OriLocalReadAddrA") - module.add(VMovB32(dst=vgpr(writer.oriLraA), src=vgpr("LocalReadAddrA"), comment="back up LRA for persistent kernel + wider local read")) - if writer.oriLraB is None: # and not kernel["DirectToVgprB"]: # no local read code if DirectToVgpr is enabled - writer.oriLraB = writer.vgprPool.checkOut(1, "OriLocalReadAddrB") - module.add(VMovB32(dst=vgpr(writer.oriLraB), src=vgpr("LocalReadAddrB"), comment="back up LRA for persistent kernel + wider local read")) + + # The code below backs up LRA at the start of the tail loop and restores at the end of tail loop. + # This would be needed if we want to support PersistentKernel in the future (without stream-k). + # Backup is not needed for stream-k since LRA is recalculated at the start of the persistent loop for the new tile or partial tile. + # Enabling the backup code would require fixing an interaction with the code that reallocates vgprs for the store code. + + # needBackupLRAddr = needRecalc or (kernel["LdsPadA"] and kernel["LdsBlockSizePerPadA"] or kernel["LdsPadB"] and kernel["LdsBlockSizePerPadB"]) and kernel["StreamK"] == 0 + + # if needBackupLRAddr: + # # need to back-up the LRA before reCalculation for wider local read (when no wlr, no need to do this) + # if writer.oriLraA is None: # and not kernel["DirectToVgprA"]: # no local read code if DirectToVgpr is enabled + # writer.oriLraA = writer.vgprPool.checkOut(1, "OriLocalReadAddrA") + # module.add(VMovB32(dst=vgpr(writer.oriLraA), src=vgpr("LocalReadAddrA"), comment="back up LRA for persistent kernel + wider local read")) + # if writer.oriLraB is None: # and not kernel["DirectToVgprB"]: # no local read code if DirectToVgpr is enabled + # writer.oriLraB = writer.vgprPool.checkOut(1, "OriLocalReadAddrB") + # module.add(VMovB32(dst=vgpr(writer.oriLraB), src=vgpr("LocalReadAddrB"), comment="back up LRA for persistent kernel + wider local read")) return module From e304cf7c0a570619c6cd5197e999fe4d2e692c44 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Fri, 6 Sep 2024 16:34:49 +0000 Subject: [PATCH 34/36] Small optimization from PR review --- tensilelite/Tensile/Components/StreamK.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index 7460420a5b..b80820526a 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -304,7 +304,7 @@ def storeBranchesCommon(self, writer, kernel, skPartialsLabel, vectorWidths, ele module.add(VReadfirstlaneB32(dst=sgpr(tmpSgpr+2), src=vgpr("Serial"), comment="Wave 0 updates flags")) module.add(SCmpEQU32(src0=sgpr(tmpSgpr+2), src1=0, comment="Check for wave 0")) module.add(SCBranchSCC0(labelName=skipFlagReset.getLabelName(), comment="Skip flag reset")) - module.add(SMovB32(dst=sgpr(tmpSgpr+2), src=0, comment="flag data")) + # (tmpSgpr+2) contains a vlue of 0, use it to reset the flag module.add(SStoreB32(src=sgpr(tmpSgpr+2), base=sgpr("AddressFlags", 2), soffset=sgpr(tmpSgpr), smem=SMEMModifiers(glc=1), comment="reset flag")) module.add(skipFlagReset) writer.sgprPool.checkIn(tmpSgpr) From 256da9bf0e253f596e155ac2f60675e450b2798c Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Fri, 6 Sep 2024 16:35:46 +0000 Subject: [PATCH 35/36] Add new test case based on race condition --- .../Tests/common/streamk/sk_bgemm_quick.yaml | 1 + .../Tests/common/streamk/sk_hgemm_quick.yaml | 1 + .../Tests/common/streamk/sk_hgemm_race.yaml | 59 +++++++++++++++++++ .../Tests/common/streamk/sk_sgemm_quick.yaml | 1 + 4 files changed, 62 insertions(+) create mode 100644 tensilelite/Tensile/Tests/common/streamk/sk_hgemm_race.yaml diff --git a/tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml b/tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml index 5e8b90170c..75c3f07f15 100644 --- a/tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml +++ b/tensilelite/Tensile/Tests/common/streamk/sk_bgemm_quick.yaml @@ -19,6 +19,7 @@ GlobalParameters: NumWarmups: 0 EnqueuesPerSync: 1 # NumBenchmarks: 10 + SleepPercent: 50 BenchmarkProblems: diff --git a/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml b/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml index e83e49cf97..cfc7c85e00 100644 --- a/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml +++ b/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_quick.yaml @@ -19,6 +19,7 @@ GlobalParameters: NumWarmups: 0 EnqueuesPerSync: 1 # NumBenchmarks: 10 + SleepPercent: 50 BenchmarkProblems: diff --git a/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_race.yaml b/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_race.yaml new file mode 100644 index 0000000000..be28408383 --- /dev/null +++ b/tensilelite/Tensile/Tests/common/streamk/sk_hgemm_race.yaml @@ -0,0 +1,59 @@ +TestParameters: + marks: [skip-gfx900, skip-gfx906, skip-gfx908, skip-gfx1010, skip-gfx1011, skip-gfx1012, skip-gfx1030, skip-gfx1100, skip-gfx1101, skip-gfx1102, skip-gfx1200, skip-gfx1201] # not supported by arch + +GlobalParameters: + MinimumRequiredVersion: 4.14.0 + SleepPercent: 50 + EnqueuesPerSync: 1 + NumWarmups: 0 + KernelTime: true + NumElementsToValidate: -1 + DataInitTypeBeta: 1 + DataInitTypeAlpha: 1 + # DataInitTypeA: 12 + # DataInitTypeB: 13 + # DataInitTypeC: 12 + # DataInitSeed: 1 + # CSVExportWinner: 1 + # CSVMergeSameProblemID: 1 + PreciseKernelTime: 0 + # Device: 0 + MaxFileName: 256 + #RotatingBufferSize: 1024 + # PrintSolutionRejectionReason: true + +BenchmarkProblems: +- - OperationType: GEMM + DataType: H + DestDataType: H + ComputeDataType: S + HighPrecisionAccumulate: true + TransposeA: False + TransposeB: True + UseBeta: True + #UseBias: 1 + Batched: True + #Activation: True + #ActivationHPA: True + - InitialSolutionParameters: + BenchmarkCommonParameters: + - KernelLanguage: [Assembly] + ForkParameters: + - 1LDSBuffer: [1] + - DepthU: [64] + - PrefetchGlobalRead: [2] + - PrefetchLocalRead: [1] + - SourceSwap: [1] + - StaggerU: [0] + - GlobalSplitU: [0] + - StreamK: [3] + - Groups: + - + - NonTemporalC: 0 + NonTemporalD: 0 + - + - MatrixInstruction: [16, 16, 16, 1, 1, 4, 8, 4, 1] # MT 256x128 + BenchmarkJoinParameters: + BenchmarkFinalParameters: + - ProblemSizes: + - Range: [ [256, 263, 2097], [256, 263, 2097], [1], [256, 263, 1045] ] diff --git a/tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml b/tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml index 2e991675bc..e228a0ce9a 100644 --- a/tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml +++ b/tensilelite/Tensile/Tests/common/streamk/sk_sgemm_quick.yaml @@ -19,6 +19,7 @@ GlobalParameters: NumWarmups: 0 EnqueuesPerSync: 1 # NumBenchmarks: 10 + SleepPercent: 50 BenchmarkProblems: From 21abe72d19404b597c024084c37a05220cd52f61 Mon Sep 17 00:00:00 2001 From: Alex Brown Date: Fri, 6 Sep 2024 21:21:17 +0000 Subject: [PATCH 36/36] Fix merge issue --- library/src/amd_detail/rocblaslt/src/tensile_host.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/library/src/amd_detail/rocblaslt/src/tensile_host.cpp b/library/src/amd_detail/rocblaslt/src/tensile_host.cpp index 99eb2a4fa8..07d6cc6106 100644 --- a/library/src/amd_detail/rocblaslt/src/tensile_host.cpp +++ b/library/src/amd_detail/rocblaslt/src/tensile_host.cpp @@ -2991,17 +2991,14 @@ std::string getKernelNameFromAlgoIndex(rocblaslt_handle handle, const rocblaslt_ std::shared_ptr deviceProp; auto adapter = get_library_and_adapter(&library, &deviceProp, handle->device); -<<<<<<< HEAD std::shared_ptr hardware; hardware = Tensile::hip::GetDevice(*deviceProp); -======= if(!library) { return std::string(); } ->>>>>>> upstream/develop int* solutionIndex = (int*)algo.data; auto solution = library->getSolutionByIndex(*hardware, *solutionIndex); return solution->kernelName; @@ -3013,17 +3010,14 @@ std::string getSolutionNameFromAlgoIndex(rocblaslt_handle handle, const rocblasl std::shared_ptr deviceProp; auto adapter = get_library_and_adapter(&library, &deviceProp, handle->device); -<<<<<<< HEAD std::shared_ptr hardware; hardware = Tensile::hip::GetDevice(*deviceProp); -======= if(!library) { return std::string(); } ->>>>>>> upstream/develop int* solutionIndex = (int*)algo.data; auto solution = library->getSolutionByIndex(*hardware , *solutionIndex); return solution->solutionName;